From a0dd202326ca59f4567c7dd9b27e9acc3a041a07 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 14:28:38 +0100 Subject: [PATCH 01/26] test(qwp): pin slot retention until worker quiescence --- ...CursorSendEngineSlotReacquisitionTest.java | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java new file mode 100644 index 00000000..2b2ce724 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -0,0 +1,218 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Engine-level regression for the shutdown hazard where + * {@link CursorSendEngine#close()} released the slot lock, closed the ring + * and watermark, and unlinked segment files while the shared + * {@link SegmentManager} worker was still mid service pass for the engine's + * ring. A replacement engine could acquire the same slot the moment the + * lock was released, after which the stale worker's abandon/trim path could + * unlink a segment path the replacement was actively writing through — + * store-and-forward data loss after restart. + *

+ * The fix makes {@code close()} run a quiescence barrier + * ({@link SegmentManager#awaitRingQuiescence}) after {@code deregister} and + * refuse to release any worker-reachable resource (ring, watermark, segment + * files, slot lock) until the barrier confirms the worker cannot touch the + * slot again. On barrier timeout the engine deliberately leaks and a later + * {@code close()} retries the cleanup. + */ +public class CursorSendEngineSlotReacquisitionTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-slot-reacq-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir == null) return; + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + + /** + * The structural guarantee: while the manager worker is provably still + * inside a service pass for the engine's ring, {@code close()} must NOT + * hand the slot to anyone else. With the quiescence barrier reverted, + * close() releases the slot lock immediately and the mid-test + * {@code SlotLock.acquire} probe succeeds — failing the test. + */ + @Test(timeout = 30_000L) + public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/slot"; + // 60 s poll: the worker only acts when explicitly woken, so the + // single pass we park below is the only pass in flight. + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + CursorSendEngine engine = null; + try { + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + + // Shared manager: ownsManager=false, so engine close() cannot + // fall back on manager.close()'s join — the per-ring barrier + // is the only protection, which is exactly what we pin here. + engine = new CursorSendEngine(slot, segSize, manager); + Assert.assertTrue("worker never reached the install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Barrier must time out fast: the worker is parked inside the + // service pass for this engine's ring. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + + // The slot must still be locked: a replacement engine (or raw + // SlotLock) acquiring it now would race the stale worker. + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock while the manager " + + "worker was still mid service pass for its ring — a " + + "replacement engine could acquire the slot and have its " + + "segment files unlinked by the stale worker"); + } catch (Exception expected) { + // good — slot retained. + } + + // Let the worker finish its pass (it abandons the spare: the + // ring was deregistered by the close attempt above). + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + + // Retry close(): the barrier now succeeds and the full cleanup + // (ring, watermark, unlink, slot lock) must complete. + engine.close(); + engine = null; + + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after a completed close", probe); + } catch (Exception e) { + throw new AssertionError("retried close() did not release the slot lock", e); + } + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (engine != null) { + try { + engine.close(); + } catch (Throwable ignored) { + } + } + if (!managerClosed) { + manager.close(); + } + } + }); + } + + /** + * Plain-positive path: after a normal close (worker quiesces promptly), + * a second engine must be able to acquire and use the same slot. + */ + @Test(timeout = 30_000L) + public void testSameSlotReacquirableAfterNormalClose() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = tmpDir + "/slot"; + CursorSendEngine first = new CursorSendEngine(slot, 4L * 1024 * 1024); + first.close(); + CursorSendEngine second = new CursorSendEngine(slot, 4L * 1024 * 1024); + try { + Assert.assertFalse("fully-drained close must leave no segments to recover", + second.wasRecoveredFromDisk()); + } finally { + second.close(); + } + }); + } + + private static void rmDirRecursive(String dir) { + if (!Files.exists(dir)) return; + long find = Files.findFirst(dir); + if (find <= 0) return; + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRecursive(child); + Files.remove(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } +} From 8f0dd45b72cb651c46ff7af4c1e65354c77e0233 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 12:17:55 +0100 Subject: [PATCH 02/26] fix(qwp): make engine close a worker-quiescence barrier before releasing slot resources SegmentManager.close() gives up after a bounded join, but CursorSendEngine.close() could not observe the incomplete shutdown: it closed the ring and watermark, unlinked segment files and released the slot flock while the worker could still be mid service pass - able to unlink a spare/trim path inside a slot directory that a replacement engine had already re-acquired (SF data loss after restart). - serviceRing now claims the entry as in-service under the manager lock and skips entries deregistered before the pass starts - new SegmentManager.awaitRingQuiescence(ring): bounded, interrupt-preserving barrier that confirms the worker can never touch the ring/slot again - CursorSendEngine.close() releases ring/watermark/segment files/slot lock only after confirmed quiescence (or a reaped owned worker); otherwise it leaks them deliberately, logs, and allows close() to be retried - a timed-out SegmentManager.close() hands pathScratch ownership to the worker, which frees it on exit - no permanent native leak when the worker outlives the join - regression: CursorSendEngineSlotReacquisitionTest (slot retained while worker mid-pass + retry completes cleanup; same-slot reacquisition after normal close) and an awaitRingQuiescence contract test --- .../client/sf/cursor/CursorSendEngine.java | 69 +++++- .../qwp/client/sf/cursor/SegmentManager.java | 200 +++++++++++++++--- .../cursor/SegmentManagerCloseRaceTest.java | 75 +++++++ 3 files changed, 309 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64ca75d0..379b5dbe 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -114,6 +114,12 @@ public final class CursorSendEngine implements QuietCloseable { // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. private volatile boolean closed; + // True once close() has run its full cleanup sequence. Stays false when + // a close attempt could not confirm manager-worker quiescence and had to + // leak the ring/watermark/slot lock — in that case a later close() call + // retries the cleanup (the worker may have exited by then). Guarded by + // the synchronized close() method; never read elsewhere. + private boolean closeCompleted; // Producer-thread-only: timestamp of the last "we're backpressured" log // line, used to throttle. Plain long is fine. private long lastBackpressureLogNs; @@ -480,7 +486,7 @@ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos @Override public synchronized void close() { - if (closed) return; + if (closed && closeCompleted) return; closed = true; // Capture drain state BEFORE closing the ring — once the ring is // closed, its accessors aren't safe to read. The active segment is @@ -492,10 +498,14 @@ public synchronized void close() { // the server has no dedup state for those messageSequences. // Memory mode has no files to unlink. // The whole close sequence runs under try/finally so the slot lock - // is ALWAYS released, even if manager/ring close or unlink throws — - // otherwise a kernel-held flock outlives the engine and the next - // sender for the same slot collides on a lock the dead engine - // never released. + // is released whenever it is safe to do so, even if manager/ring + // close or unlink throws — otherwise a kernel-held flock outlives + // the engine and the next sender for the same slot collides on a + // lock the dead engine never released. The one deliberate exception + // is a manager worker that failed to quiesce: releasing the lock + // then would let a replacement engine acquire the slot while the + // stale worker can still create/unlink segment paths inside it. + boolean workerQuiescent = false; try { // "Fully drained" includes BOTH the obvious case (every published // FSN has been acked) AND the never-published case (publishedFsn @@ -504,9 +514,15 @@ public synchronized void close() { // recreates a fresh sf-initial.sfa — would otherwise leave that // fresh empty file behind, the next scanner finds it, adopts the // slot again, and the cycle repeats forever (M6). - boolean fullyDrained = sfDir != null - && (ring.publishedFsn() < 0 - || ring.ackedFsn() >= ring.publishedFsn()); + // Own try/catch so sabotaged/broken ring state cannot skip the + // quiescence barrier below or the slot-lock release in finally. + boolean fullyDrained = false; + try { + fullyDrained = sfDir != null + && (ring.publishedFsn() < 0 + || ring.ackedFsn() >= ring.publishedFsn()); + } catch (Throwable ignored) { + } // Each cleanup step in its own try/catch so a single failure // doesn't strand later cleanups — mirrors the constructor's // catch block. Without this, a throw from manager.deregister @@ -517,11 +533,41 @@ public synchronized void close() { manager.deregister(ring); } catch (Throwable ignored) { } + // Quiescence barrier. deregister alone only removes the entry + // from the registry — the worker may still be mid service pass + // for this ring (creating a spare file, trimming, unlinking). + // Releasing the ring, watermark, segment files, or the slot lock + // while that pass is in flight lets the stale worker unlink a + // path that a replacement engine — which can acquire the slot + // the moment the lock is released — is actively writing through: + // store-and-forward data loss after restart. Wait for confirmed + // quiescence before touching anything the worker can reach. + try { + workerQuiescent = manager.awaitRingQuiescence(ring); + } catch (Throwable ignored) { + } if (ownsManager) { try { manager.close(); } catch (Throwable ignored) { } + if (!workerQuiescent) { + // manager.close() joins the worker; a reaped (or + // never-started) worker is an even stronger barrier + // than per-ring quiescence. + try { + workerQuiescent = manager.isWorkerReaped(); + } catch (Throwable ignored) { + } + } + } + if (!workerQuiescent) { + LOG.error("SF manager worker did not quiesce during engine close; leaking the " + + "ring, watermark and slot lock so a stale worker cannot corrupt a " + + "future engine on slot {}. The kernel releases the slot flock on " + + "process exit; close() may be invoked again to retry cleanup once " + + "the worker has exited.", sfDir == null ? "" : sfDir); + return; } try { ring.close(); @@ -551,8 +597,13 @@ public synchronized void close() { } catch (Throwable ignored) { } } + closeCompleted = true; } finally { - if (slotLock != null) { + // Gate on quiescence: releasing the flock while a stale worker + // can still touch this slot directory hands the slot to a new + // engine that the worker may then corrupt. Leaking the fd is + // the safe failure mode — the kernel drops it on process exit. + if (workerQuiescent && slotLock != null) { try { slotLock.close(); } catch (Throwable ignored) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 3b7cff60..6548fdef 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -100,15 +100,34 @@ public final class SegmentManager implements QuietCloseable { // registered flag check inside the trim block closes for watermark writes // and totalBytes accounting. private volatile Runnable beforeTrimSyncHook; + // Entry currently being serviced by the worker thread, or null when the + // worker is between service passes (or not running). Guarded by + // {@link #lock}; cleared with lock.notifyAll() so awaitRingQuiescence can + // block until an in-flight pass for a just-deregistered ring finishes. + private RingEntry inService; private long lastDiskFullLogNs; private volatile boolean running; + // pathScratch free-exactly-once coordination between a timed-out close() + // and the worker's exit path. All three are guarded by {@link #lock}. + // When close() gives up on the join while the worker loop has not yet + // exited, it hands scratch ownership to the worker + // (scratchHandedToWorker=true) and the worker frees the buffer in its + // exit block; in every other case close() frees it. Without the handoff, + // a worker that outlives the bounded join leaks the native scratch + // buffer forever, because nobody retries manager cleanup after close() + // returns. + private boolean scratchFreed; + private boolean scratchHandedToWorker; + private boolean workerLoopExited; // Total bytes currently allocated across every segment owned by every // registered ring (active + sealed + hot-spare). Mutated by the manager // thread on provision/trim and by register/deregister callers under // {@link #lock}; the lock covers both paths so the counter stays // consistent across registration boundaries. private long totalBytes; - private long workerJoinTimeoutMillis = WORKER_JOIN_TIMEOUT_MILLIS; + // volatile: read by awaitRingQuiescence() from arbitrary caller threads + // while the @TestOnly setter may run on another. + private volatile long workerJoinTimeoutMillis = WORKER_JOIN_TIMEOUT_MILLIS; // volatile because wakeWorker() reads workerThread without holding the // monitor; the synchronized start()/close() pair handles the // start-vs-close ordering. @@ -194,18 +213,100 @@ public synchronized void close() { } } if (t.isAlive()) { - LOG.warn("SegmentManager worker did not stop before close wait completed; " - + "leaving worker-owned resources allocated"); - return; + synchronized (lock) { + if (!workerLoopExited) { + // Hand pathScratch ownership to the worker: its exit + // block frees the buffer under the same lock, so the + // native allocation is reclaimed even though this + // close() could not confirm termination. workerThread + // stays set so isWorkerReaped() reports the incomplete + // shutdown and a later close() can retry the join. + scratchHandedToWorker = true; + LOG.warn("SegmentManager worker did not stop before close wait completed; " + + "worker frees its native scratch buffer on exit"); + return; + } + } + // The worker loop has already run its exit block; the thread + // is at most a few instructions from terminating and can no + // longer touch manager state. Fall through and reap it. } workerThread = null; } // Free the rotation-path native scratch buffer only after worker - // termination has been observed. The worker is the only thread that - // touches the buffer, but close() uses a bounded join; if the worker is - // still alive, leaking this one scratch allocation is safer than freeing - // native memory it may still read or write. - pathScratch.close(); + // termination (or worker-loop exit) has been observed. The worker is + // the only thread that touches the buffer; the scratchFreed flag + // (shared with the worker's exit block) makes the free exactly-once + // no matter which side runs last. + synchronized (lock) { + if (!scratchFreed) { + scratchFreed = true; + pathScratch.close(); + } + } + } + + /** + * Quiescence barrier for {@link #deregister(SegmentRing)}. Blocks until + * the worker thread is provably no longer executing a service pass for + * {@code ring}, or the worker-join timeout elapses. After this returns + * {@code true}, the worker will never again touch the ring, its + * watermark, or path names under its slot directory: deregister has + * removed the entry from the registry, a stale snapshot entry that has + * not started its pass is skipped by the registration check at the top + * of {@link #serviceRing(RingEntry)}, and this method has observed the + * end of any in-flight pass. Only then may the caller release dependent + * resources (ring, watermark, segment files, slot lock). + *

+ * Returns {@code true} immediately when no worker is running or when + * called from the worker thread itself (test hooks inject deregister + * calls there; waiting would self-deadlock). Returns {@code false} when + * the in-flight pass did not finish within the timeout — the caller must + * treat the worker as still live and leak rather than release. + *

+ * A pending caller interrupt is preserved but does not abort the wait, + * mirroring {@link #close()}. + */ + public boolean awaitRingQuiescence(SegmentRing ring) { + Thread t = workerThread; + if (t == null || t == Thread.currentThread()) { + return true; + } + long deadlineNanos = System.nanoTime() + workerJoinTimeoutMillis * 1_000_000L; + boolean interrupted = Thread.interrupted(); + try { + synchronized (lock) { + while (inService != null && inService.ring == ring) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0) { + return false; + } + try { + // Round up so a sub-millisecond remainder still waits + // instead of spinning through wait(0) == wait-forever. + lock.wait(Math.max(1L, remainingNanos / 1_000_000L)); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + } + return true; + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + /** + * True when no manager worker thread can be running: either + * {@link #start()} was never called, or a {@link #close()} confirmed + * worker termination and reaped the thread. Owners use this as a + * stronger fallback barrier when {@link #awaitRingQuiescence(SegmentRing)} + * times out but a subsequent {@code close()} join succeeded. + */ + public synchronized boolean isWorkerReaped() { + return workerThread == null; } /** @@ -213,6 +314,13 @@ public synchronized void close() { * created after this returns, but already-installed spares stay with * the ring (the ring closes them on its own {@link SegmentRing#close}). * Idempotent; safe to call from any thread. + *

+ * Non-blocking: a worker service pass already in flight for this ring + * may still be running when this returns. Callers about to release + * resources the worker can reach (the ring itself, its watermark, its + * segment files, or the slot lock guarding its directory) MUST follow + * up with {@link #awaitRingQuiescence(SegmentRing)} and only release on + * a {@code true} result. */ public void deregister(SegmentRing ring) { synchronized (lock) { @@ -412,6 +520,32 @@ private String nextSparePath(String dir) { } private void serviceRing(RingEntry e) { + // Claim the entry as in-service so deregister-side quiescence + // barriers (awaitRingQuiescence) can wait for this pass to finish. + // A stale snapshot entry deregistered before the pass starts is + // skipped entirely: the deregistering thread may already be + // releasing the ring / watermark / slot resources, so the worker + // must not touch them at all. The registered check and the + // in-service claim are atomic under `lock` — deregister flips + // `registered` under the same lock, so it either prevents this + // pass or the barrier observes it via `inService`. + synchronized (lock) { + if (!e.registered) { + return; + } + inService = e; + } + try { + serviceRing0(e); + } finally { + synchronized (lock) { + inService = null; + lock.notifyAll(); + } + } + } + + private void serviceRing0(RingEntry e) { // 1. Provision a hot spare if the ring needs one AND we have headroom // under the disk-total cap. Cap check is per-tick; if we're capped // here, the ring stays in BACKPRESSURE_NO_SPARE until trim (step 2) @@ -571,26 +705,40 @@ private void serviceRing(RingEntry e) { } private void workerLoop() { - while (running) { - // Snapshot the rings under the lock so we don't hold it through the - // (potentially slow) syscalls during creation/unlink. ringSnapshot - // is a thread-confined field — no per-tick allocation. - ringSnapshot.clear(); - synchronized (lock) { - for (int i = 0, n = rings.size(); i < n; i++) { - ringSnapshot.add(rings.getQuick(i)); + try { + while (running) { + // Snapshot the rings under the lock so we don't hold it through the + // (potentially slow) syscalls during creation/unlink. ringSnapshot + // is a thread-confined field — no per-tick allocation. + ringSnapshot.clear(); + synchronized (lock) { + for (int i = 0, n = rings.size(); i < n; i++) { + ringSnapshot.add(rings.getQuick(i)); + } } - } - for (int i = 0, n = ringSnapshot.size(); i < n; i++) { + for (int i = 0, n = ringSnapshot.size(); i < n; i++) { + if (!running) break; + serviceRing(ringSnapshot.getQuick(i)); + } + // Drop strong refs so a deregistered ring becomes collectable + // before the next tick (otherwise the snapshot pins it for up + // to pollNanos after deregister). + ringSnapshot.clear(); if (!running) break; - serviceRing(ringSnapshot.getQuick(i)); + LockSupport.parkNanos(pollNanos); + } + } finally { + // If a timed-out close() abandoned the reap, it handed + // pathScratch ownership to this thread (see close()). Freeing it + // here reclaims the native buffer even when the worker outlives + // every close() attempt — nobody else retries manager cleanup. + synchronized (lock) { + workerLoopExited = true; + if (scratchHandedToWorker && !scratchFreed) { + scratchFreed = true; + pathScratch.close(); + } } - // Drop strong refs so a deregistered ring becomes collectable - // before the next tick (otherwise the snapshot pins it for up - // to pollNanos after deregister). - ringSnapshot.clear(); - if (!running) break; - LockSupport.parkNanos(pollNanos); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index 3d7c6a7c..4e62f8ca 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -204,6 +204,81 @@ public void testCloseDoesNotFreePathScratchWhenWorkerStillAlive() throws Excepti }); } + /** + * Pins the {@link SegmentManager#awaitRingQuiescence} contract that + * {@code CursorSendEngine.close()} depends on: + *

+ */ + @Test(timeout = 15_000L) + public void testAwaitRingQuiescenceBlocksWhileServicePassInFlight() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/quiesce-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + try { + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + manager.deregister(ring); + manager.setWorkerJoinTimeoutMillis(50L); + Thread.currentThread().interrupt(); + Assert.assertFalse( + "awaitRingQuiescence returned true while the worker was parked " + + "inside the service pass for this ring", + manager.awaitRingQuiescence(ring)); + Assert.assertTrue("awaitRingQuiescence must preserve the caller's interrupt", + Thread.interrupted()); + + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + Assert.assertTrue( + "awaitRingQuiescence must return true once the in-flight pass finished", + manager.awaitRingQuiescence(ring)); + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + Thread.interrupted(); + manager.close(); + } + ring.close(); + } + }); + } + @Test(timeout = 15_000L) public void testInterruptedCallerDoesNotAbandonReapableWorker() throws Exception { TestUtils.assertMemoryLeak(() -> { From d976fd5db4176df99db78e5ea8668e49917103ec Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 15:16:00 +0100 Subject: [PATCH 03/26] fix(qwp): preserve slot ownership after incomplete close --- .../qwp/client/QwpWebSocketSender.java | 57 ++++--- .../client/sf/cursor/CursorSendEngine.java | 39 +++-- .../qwp/client/sf/cursor/SegmentManager.java | 13 ++ .../io/questdb/client/impl/SenderPool.java | 35 ++--- .../client/SlotLockReleasedContractTest.java | 143 +++++++++++++++++- ...CursorSendEngineSlotReacquisitionTest.java | 36 +++++ 6 files changed, 271 insertions(+), 52 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d1744065..c0998bcf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -278,10 +278,10 @@ public class QwpWebSocketSender implements Sender { private boolean ownsCursorEngine; private long pendingBytes; // Set true by close() once the SF slot flock has been released (the normal - // teardown path). Stays false if close() bailed early with the I/O thread - // still running -- then cursorEngine.close() never ran and the flock is - // still held, so the owning pool MUST keep the slot reserved rather than - // hand the still-locked dir to the next borrow ("sf slot already in use"). + // teardown path). Stays false if an I/O or manager worker did not stop and + // cursorEngine retained the flock, so the owning pool MUST keep the slot + // reserved rather than hand the still-locked dir to the next borrow + // ("sf slot already in use"). private boolean slotLockReleased; private int pendingRowCount; private SenderProgressDispatcher progressDispatcher; @@ -1038,7 +1038,10 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { * replaying frames get a 2.5s grace window plus a 0.5s stop window * — worst case ~3s when a drainer sits in a blocking native * connect (15s background deadline) and must be abandoned to exit - * on its own. + * on its own; + *
  • SF manager stop: normally immediate, bounded by 5s when its worker + * is stuck in a filesystem operation. On timeout the slot remains + * locked rather than being exposed to a stale worker.
    • * */ @Override @@ -1189,15 +1192,23 @@ public void close() { // the failed close() and now — then closing here is safe. if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null && !cursorSendLoop.delegateEngineClose()) { + CursorSendEngine engine = cursorEngine; try { - cursorEngine.close(); + engine.close(); } catch (Throwable t) { LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); terminalError = captureCloseError(terminalError, t); } - cursorEngine = null; - ownsCursorEngine = false; - slotLockReleased = true; + if (engine.isCloseCompleted()) { + cursorEngine = null; + ownsCursorEngine = false; + slotLockReleased = true; + } else { + // Preserve the engine and report the retained flock. + // Sender.close() is idempotent, so this is a deliberate + // safe leak until process exit rather than a retry path. + slotLockReleased = false; + } } rethrowTerminal(terminalError); return; @@ -1231,19 +1242,27 @@ public void close() { } if (ownsCursorEngine && cursorEngine != null) { + CursorSendEngine engine = cursorEngine; try { - cursorEngine.close(); + engine.close(); } catch (Throwable t) { LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); terminalError = captureCloseError(terminalError, t); } - cursorEngine = null; - ownsCursorEngine = false; + if (engine.isCloseCompleted()) { + cursorEngine = null; + ownsCursorEngine = false; + slotLockReleased = true; + } else { + // The manager worker did not quiesce. Preserve ownership + // and report the retained flock so pools retire this slot. + // Repeated Sender.close() calls remain no-ops by contract. + slotLockReleased = false; + } + } else { + // This sender owns no cursor engine holding an SF flock. + slotLockReleased = true; } - // Past the ioThreadStopped guard => cursorEngine.close() ran and - // released the SF flock in its finally (or this sender owned no - // engine holding one). Signal the pool it may reuse the slot. - slotLockReleased = true; // Shutdown order: dispatcher last, after the I/O loop has stopped // producing into it. close() drains pending entries with a short @@ -1288,9 +1307,9 @@ public void close() { /** * True once {@link #close()} has released the store-and-forward slot - * flock. False means close() leaked the still-running I/O thread (and its - * resources), so the flock is still held; the owning pool must keep the - * slot index reserved instead of reusing the still-locked slot dir. + * flock. False means an I/O or manager worker did not stop and close() + * retained the lock and worker-reachable resources; the owning pool must + * keep the slot index reserved instead of reusing the still-locked dir. */ public boolean isSlotLockReleased() { return slotLockReleased; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 379b5dbe..54f675f9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -540,25 +540,26 @@ public synchronized void close() { // while that pass is in flight lets the stale worker unlink a // path that a replacement engine — which can acquire the slot // the moment the lock is released — is actively writing through: - // store-and-forward data loss after restart. Wait for confirmed - // quiescence before touching anything the worker can reach. - try { - workerQuiescent = manager.awaitRingQuiescence(ring); - } catch (Throwable ignored) { - } + // store-and-forward data loss after restart. if (ownsManager) { + // Stopping and reaping a private manager is a stronger barrier + // than waiting for this ring alone. Do it directly so a stuck + // worker consumes at most one workerJoinTimeoutMillis budget, + // rather than one here and a second one in manager.close(). try { manager.close(); } catch (Throwable ignored) { } - if (!workerQuiescent) { - // manager.close() joins the worker; a reaped (or - // never-started) worker is an even stronger barrier - // than per-ring quiescence. - try { - workerQuiescent = manager.isWorkerReaped(); - } catch (Throwable ignored) { - } + try { + workerQuiescent = manager.isWorkerReaped(); + } catch (Throwable ignored) { + } + } else { + // A shared manager must keep serving its other rings, so wait + // only for the deregistered ring's current pass to finish. + try { + workerQuiescent = manager.awaitRingQuiescence(ring); + } catch (Throwable ignored) { } } if (!workerQuiescent) { @@ -613,6 +614,16 @@ public synchronized void close() { } } + /** + * Whether {@link #close()} completed all cleanup, including releasing the + * SF slot lock. A false value after close means manager-worker quiescence + * could not be confirmed and the worker-reachable resources were retained + * deliberately. Owners must not report or reuse the slot in that case. + */ + public synchronized boolean isCloseCompleted() { + return closeCompleted; + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 6548fdef..06e77371 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -93,6 +93,10 @@ public final class SegmentManager implements QuietCloseable { // but before ownership/accounting commit. Callers may inject a deregister // or hold this stale worker snapshot while caller-side cleanup runs. private volatile Runnable beforeInstallSyncHook; + // Test seam: records entry into the per-ring quiescence wait. Null in + // production; owned-engine close tests use it to prove they take only the + // stronger whole-manager join path, not two sequential timeout budgets. + private volatile Runnable beforeRingQuiescenceAwaitHook; // Test seam: runs on the worker thread just before the trim block's // synchronized(lock) entry. Null in production; only // SegmentManagerTrimDeregisterRaceTest installs it, to deterministically @@ -268,6 +272,10 @@ public synchronized void close() { * mirroring {@link #close()}. */ public boolean awaitRingQuiescence(SegmentRing ring) { + Runnable hook = beforeRingQuiescenceAwaitHook; + if (hook != null) { + hook.run(); + } Thread t = workerThread; if (t == null || t == Thread.currentThread()) { return true; @@ -410,6 +418,11 @@ public void setBeforeInstallSyncHook(Runnable hook) { this.beforeInstallSyncHook = hook; } + @TestOnly + public void setBeforeRingQuiescenceAwaitHook(Runnable hook) { + this.beforeRingQuiescenceAwaitHook = hook; + } + @TestOnly public void setBeforeTrimSyncHook(Runnable hook) { this.beforeTrimSyncHook = hook; diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 1cc214b4..31e50f31 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -171,7 +171,7 @@ public final class SenderPool implements AutoCloseable { // down on another thread. Guarded by lock. private int pendingLeaseTeardowns; // Slots whose delegate close() returned with the SF flock still held - // (the I/O thread refused to stop). Permanently consumed: the index is + // because an I/O or manager worker did not stop. Permanently consumed: // never freed and never reused, so no borrow ever hands out a still- // locked slot dir. Counted in the cap check so the lost capacity is // accounted for. Guarded by lock; only ever ticks for SF slots. @@ -523,15 +523,15 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { // leakedSlots (retire) or the freed index carries the cap math. recoveringSlots--; if (flockHeld[0]) { - // close() bailed early with the I/O thread still running and - // the flock still held. Retire the slot permanently (mirror + // close() retained the flock because an I/O or manager + // worker did not stop. Retire the slot permanently (mirror // discardBroken/reapIdle): keep slotInUse[i] set and count it // in leakedSlots so the borrow() cap math accounts for the // lost capacity and no later borrow ever reuses the // still-locked dir. leakedSlots++; LOG.warn("startup SF recovery: slot {} retired permanently: delegate close() returned with " - + "the flock still held (I/O thread refused to stop); pool capacity reduced by 1, " + + "the flock still held (I/O or manager worker did not stop); pool capacity reduced by 1, " + "now {} of {} usable [leakedSlots={}]", i, maxSize - leakedSlots, maxSize, leakedSlots); } else { @@ -580,12 +580,13 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { if (flockHeld[0]) { // Out of the pool's [0, maxSize) capacity range: there is no // slotInUse entry to retire and no future borrow targets this - // dir, so a still-held flock only leaks this recoverer's I/O - // thread (a best-effort teardown loss, logged). Crucially we do + // dir, so a still-held flock only leaks this recoverer's + // worker-reachable resources (a best-effort teardown loss, + // logged). Crucially we do // NOT touch leakedSlots -- that would wrongly shrink the // in-range pool capacity. LOG.warn("startup SF recovery: out-of-range slot {} closed with the flock still held " - + "(I/O thread refused to stop); its data is durable on disk for a later attempt", + + "(I/O or manager worker did not stop); its data is durable on disk for a later attempt", slotPath); } if (stopScan) { @@ -609,7 +610,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { * * @param flockHeld single-element out-param set to {@code true} iff a * recoverer was built and its {@code close()} returned with - * the flock still held (the I/O thread refused to stop) + * the flock still held because a worker did not stop * @return {@code true} if a build/drain failure occurred that will very * likely repeat for every remaining slot, so the caller should stop scanning */ @@ -618,8 +619,8 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, flockHeld[0] = false; // Hoisted so the flock check after the try can consult it: // createRecoverer() takes the slot flock on -slotIndex, and - // delegate().close() can early-return with the I/O thread still running - // (flock still held). + // delegate().close() can retain it when an I/O or manager worker does + // not stop. SenderSlot recoverer = null; boolean stopScan = false; try { @@ -1067,7 +1068,7 @@ public void reapIdle() { } // Return reserved SF slot indices to the free set -- but only for // slots whose delegate confirmed the flock was released. A slot - // left locked (I/O thread refused to stop) is retired permanently. + // left locked because a worker did not stop is retired permanently. if (storeAndForward) { lock.lock(); try { @@ -1107,8 +1108,8 @@ public int totalSize() { /** * Snapshot of the number of SF slots permanently retired because a - * delegate {@code close()} returned with the slot flock still held (the - * I/O thread refused to stop). Each leaked slot permanently lowers the + * delegate {@code close()} returned with the slot flock still held after + * an I/O or manager worker did not stop. Each leaked slot permanently lowers the * pool's effective capacity ({@code maxSize - leakedSlotCount()}). A * non-zero, growing value explains a pool that has started timing out * every {@code borrow()}. For metrics and tests. @@ -1271,7 +1272,7 @@ private void freeSlotIndex(int idx) { * non-QWP delegate never holds an SF flock, so it is always treated as * released. A {@link QwpWebSocketSender} reports it via * {@link QwpWebSocketSender#isSlotLockReleased()} -- false means close() - * bailed early with the I/O thread still running and the flock still held. + * retained the flock because an I/O or manager worker did not stop. */ private static boolean flockReleased(SenderSlot s) { Sender d = s.delegate(); @@ -1281,8 +1282,8 @@ private static boolean flockReleased(SenderSlot s) { /** * Reclaims one SF slot after its delegate's {@code close()} has been * attempted. When the flock was released the index returns to the free - * set; when {@code close()} returned with the flock still held (the I/O - * thread refused to stop) the slot is retired permanently -- + * set; when {@code close()} returned with the flock still held because an + * I/O or manager worker did not stop, the slot is retired permanently -- * {@code leakedSlots++} and {@code slotInUse[idx]} stays set -- so the cap * math accounts for the lost capacity and no later borrow ever reuses the * still-locked dir. Either way {@code closingSlots} is decremented. @@ -1304,7 +1305,7 @@ private boolean reclaimSlot(SenderSlot s, String context) { } leakedSlots++; LOG.warn("SF slot {} retired permanently{}: delegate close() returned with the flock still held " + - "(I/O thread refused to stop); pool capacity reduced by 1, now {} of {} usable [leakedSlots={}]", + "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable [leakedSlots={}]", s.slotIndex(), context, maxSize - leakedSlots, maxSize, leakedSlots); return false; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index 85522e9d..97ae9dbe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -26,7 +26,12 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -36,8 +41,12 @@ import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; /** @@ -46,8 +55,8 @@ *
      *
    • Happy path — a clean {@code close()} that winds the I/O thread * down reports {@code isSlotLockReleased() == true}.
      • - *
    • Leak path — a {@code close()} that early-returns because the I/O - * thread refused to stop ({@code ioThreadStopped == false}) reports + *
    • Leak path — a {@code close()} that retains the lock because an + * I/O or manager worker did not stop reports * {@code isSlotLockReleased() == false}.
      • *
    * The pool's {@code flockReleased(s)} treats {@code false} as "flock still held, @@ -178,6 +187,114 @@ public void testSlotLockNotReleasedWhenIoThreadRefusesToStop() throws Exception }); } + /** + * Manager-worker leak path: engine close retains the slot while a shared + * manager is still inside a service pass. The sender must expose that + * retained flock to SenderPool. Repeated sender close calls remain no-ops; + * the test cleans the deliberately retained engine up directly. + */ + @Test(timeout = 30_000L) + public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-manager-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + CursorSendEngine engine = null; + QwpWebSocketSender wss = null; + boolean managerClosed = false; + try { + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + engine = new CursorSendEngine(slot, segSize, manager); + Assert.assertTrue("worker never reached install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + wss = QwpWebSocketSender.createForTesting("localhost", 1); + wss.setCursorEngine(engine, true); + manager.setWorkerJoinTimeoutMillis(50L); + wss.close(); + + Assert.assertFalse("sender reported a retained flock as released", + wss.isSlotLockReleased()); + Assert.assertFalse("engine close must remain incomplete while worker is in service", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot became acquirable while manager worker was still in service"); + } catch (Exception expected) { + // good — the incomplete close retained the flock. + } + + // Sender.close() is idempotent: a second call must not retry + // or change ownership while the manager worker is still live. + wss.close(); + Assert.assertFalse("repeated close changed retained-flock reporting", + wss.isSlotLockReleased()); + Assert.assertFalse("repeated close unexpectedly retried engine cleanup", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + // Test-only cleanup through the retained local reference. The + // sender conservatively keeps reporting false, as required by + // a pool that may already have retired this slot. + engine.close(); + Assert.assertTrue("direct cleanup did not complete after worker exit", + engine.isCloseCompleted()); + Assert.assertFalse("sender must not revise the result after close returned", + wss.isSlotLockReleased()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after direct cleanup", probe); + } + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (wss != null) { + try { + wss.close(); + } catch (Throwable ignored) { + } + } + if (engine != null && !engine.isCloseCompleted()) { + try { + engine.close(); + } catch (Throwable ignored) { + } + } + if (!managerClosed) { + manager.close(); + } + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + // ------------------------------------------------------------------ utils private static void freeFieldQuietly(Object target, String name) { @@ -192,6 +309,28 @@ private static void freeFieldQuietly(Object target, String name) { } } + private static void rmDirRecursive(String dir) { + if (!Files.exists(dir)) return; + long find = Files.findFirst(dir); + if (find <= 0) return; + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRecursive(child); + Files.remove(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + private static T readField(Object target, String name, Class type) throws Exception { Class cls = target.getClass(); while (cls != null) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 2b2ce724..5ab33da7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -35,6 +35,7 @@ import org.junit.Before; import org.junit.Test; +import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -123,6 +124,8 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { // service pass for this engine's ring. manager.setWorkerJoinTimeoutMillis(50L); engine.close(); + Assert.assertFalse("incomplete close must remain observable to the owner", + engine.isCloseCompleted()); // The slot must still be locked: a replacement engine (or raw // SlotLock) acquiring it now would race the stale worker. @@ -145,6 +148,8 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { // Retry close(): the barrier now succeeds and the full cleanup // (ring, watermark, unlink, slot lock) must complete. engine.close(); + Assert.assertTrue("retried close must report complete cleanup", + engine.isCloseCompleted()); engine = null; try (SlotLock probe = SlotLock.acquire(slot)) { @@ -174,6 +179,31 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { }); } + /** + * An engine that owns its manager must use the whole-manager stop/join as + * its only quiescence barrier. Calling the per-ring barrier first would + * give a stuck worker two independent timeout budgets. + */ + @Test(timeout = 30_000L) + public void testOwnedManagerCloseSkipsPerRingQuiescenceWait() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = tmpDir + "/owned-slot"; + CursorSendEngine engine = new CursorSendEngine(slot, 4L * 1024 * 1024); + SegmentManager manager = readManager(engine); + AtomicBoolean perRingAwaited = new AtomicBoolean(); + try { + manager.setBeforeRingQuiescenceAwaitHook(() -> perRingAwaited.set(true)); + engine.close(); + Assert.assertTrue("owned engine close did not complete", engine.isCloseCompleted()); + Assert.assertFalse("owned engine close spent a separate per-ring wait budget", + perRingAwaited.get()); + } finally { + manager.setBeforeRingQuiescenceAwaitHook(null); + engine.close(); + } + }); + } + /** * Plain-positive path: after a normal close (worker quiesces promptly), * a second engine must be able to acquire and use the same slot. @@ -194,6 +224,12 @@ public void testSameSlotReacquirableAfterNormalClose() throws Exception { }); } + private static SegmentManager readManager(CursorSendEngine engine) throws Exception { + Field field = CursorSendEngine.class.getDeclaredField("manager"); + field.setAccessible(true); + return (SegmentManager) field.get(engine); + } + private static void rmDirRecursive(String dir) { if (!Files.exists(dir)) return; long find = Files.findFirst(dir); From d3b61478ddb57256ffe70c209fab03de19eaaebf Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 17:08:44 +0100 Subject: [PATCH 04/26] test(qwp): pin slot retention on the owned-manager close path Every production CursorSendEngine (Sender.build, BackgroundDrainer, QwpWebSocketSender.connect) owns its SegmentManager, so close() takes the manager.close() + isWorkerReaped() branch - yet all deterministic retention tests exercised the test-only shared-manager branch (awaitRingQuiescence). A regression confined to the owned path - reporting quiescence unconditionally, or isWorkerReaped() returning true while the worker is alive - would have gone green through the whole suite and silently reintroduced the SF-data-loss hazard on the only path production runs. testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass builds the production shape (2-arg ctor, private owned manager), waits for the initial hot-spare install so the park hook can neither be missed nor fire early, rotates onto the spare to force the worker back into an install pass, parks it there, and drives close() with a 50 ms join budget. It asserts the incomplete close stays observable (isCloseCompleted() == false), the slot flock is retained (SlotLock.acquire throws), and a retried close after worker release completes the full cleanup and frees the slot. Mutation-verified red on all three reverts: - owned branch forcing workerQuiescent = true (only this test catches it) - finally gate reverted to unconditional slotLock.close() - SegmentManager.isWorkerReaped() returning true while the worker is alive (previously zero coverage anywhere) --- ...CursorSendEngineSlotReacquisitionTest.java | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 5ab33da7..9afb86eb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -27,8 +27,11 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.After; import org.junit.Assert; @@ -179,6 +182,120 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { }); } + /** + * Owned-manager twin of {@link #testCloseRetainsSlotWhileWorkerIsMidServicePass}: + * the ONLY construction shape production uses (Sender.build, BackgroundDrainer, + * QwpWebSocketSender.connect all own their manager). The owned close path does + * not run the per-ring barrier at all — it relies on {@code manager.close()}'s + * bounded join and the {@code isWorkerReaped()} check. If that check regressed + * to report quiescence unconditionally (or {@code isWorkerReaped()} itself + * returned true while the worker is alive), close() would release the slot + * lock mid service pass and the shared-manager tests would stay green — this + * test is the red gate for the production path. + *

    + * Determinism: the owned manager starts inside the engine ctor (1 ms poll), + * so its first spare-install pass races test setup. We first wait until the + * initial hot spare is installed — after that the worker cannot enter another + * install pass until a rotation consumes the spare, so the park hook installed + * afterwards can neither be missed nor fire early. Two appends then fill the + * active segment and rotate onto the spare; the worker's next poll tick + * re-enters the install pass and parks in the hook. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-parked-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: let the worker finish the initial spare install so + // the hook below can only fire on the rotation-triggered pass. + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: one frame fills the active segment exactly; the + // second forces rotation onto the spare. needsHotSpare() is + // true again, so the worker's next tick parks in the hook. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: owned close with the worker provably mid service + // pass. manager.close()'s 50 ms join times out, the worker is + // not reaped, and close() must retain every worker-reachable + // resource — above all the slot flock. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("incomplete owned close must remain observable to the owner", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("owned engine.close() released the slot lock while its manager " + + "worker was still mid service pass — a replacement engine could " + + "acquire the slot and have its segment files unlinked by the " + + "stale worker (the production SF-data-loss hazard)"); + } catch (Exception expected) { + // good — slot retained. + } + + // Phase 4: release the worker (it abandons the spare: the ring + // was deregistered by the close attempt) and retry. The join + // now reaps the worker and the full cleanup must complete. + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + engine.close(); + Assert.assertTrue("retried owned close must report complete cleanup", + engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after a completed close", probe); + } catch (Exception e) { + throw new AssertionError("retried owned close() did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + /** * An engine that owns its manager must use the whole-manager stop/join as * its only quiescence barrier. Calling the per-ring barrier first would @@ -230,6 +347,12 @@ private static SegmentManager readManager(CursorSendEngine engine) throws Except return (SegmentManager) field.get(engine); } + private static SegmentRing readRing(CursorSendEngine engine) throws Exception { + Field field = CursorSendEngine.class.getDeclaredField("ring"); + field.setAccessible(true); + return (SegmentRing) field.get(engine); + } + private static void rmDirRecursive(String dir) { if (!Files.exists(dir)) return; long find = Files.findFirst(dir); From 7cf96122fb3ad39f1cf532b1ad255ee9f780d0db Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 19:19:22 +0100 Subject: [PATCH 05/26] perf(qwp): wake quiescence waiters only when one is parked serviceRing's per-pass finally called lock.notifyAll() unconditionally - with the default 1 ms poll that is ~1000 wakeups/sec per registered ring on the production worker, paid for a barrier (awaitRingQuiescence) that production never takes: all three production constructions own their manager, so close() goes through manager.close()+isWorkerReaped() and never parks on the lock. - new quiescenceWaiters count, incremented/decremented around the awaitRingQuiescence wait loop under the same lock as the worker's check - no lost-wakeup window, and the timed wait remains a fallback - the per-pass finally notifies only when quiescenceWaiters > 0; in steady state it never fires - notifyAll (not notify) retained when a waiter exists: with a shared manager, distinct waiters may await different rings --- .../qwp/client/sf/cursor/SegmentManager.java | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 06e77371..726374d6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -109,6 +109,14 @@ public final class SegmentManager implements QuietCloseable { // {@link #lock}; cleared with lock.notifyAll() so awaitRingQuiescence can // block until an in-flight pass for a just-deregistered ring finishes. private RingEntry inService; + // Number of threads currently parked in awaitRingQuiescence()'s wait + // loop. Guarded by {@link #lock}. The worker's per-pass finally block + // only calls lock.notifyAll() when this is > 0, so the steady-state + // production path (no quiescence barrier in flight) never notifies. + // Both sides mutate/check under the same lock, so there is no lost + // wakeup: a waiter that increments after the worker's check re-reads + // inService before waiting and sees the pass already finished. + private int quiescenceWaiters; private long lastDiskFullLogNs; private volatile boolean running; // pathScratch free-exactly-once coordination between a timed-out close() @@ -284,18 +292,23 @@ public boolean awaitRingQuiescence(SegmentRing ring) { boolean interrupted = Thread.interrupted(); try { synchronized (lock) { - while (inService != null && inService.ring == ring) { - long remainingNanos = deadlineNanos - System.nanoTime(); - if (remainingNanos <= 0) { - return false; - } - try { - // Round up so a sub-millisecond remainder still waits - // instead of spinning through wait(0) == wait-forever. - lock.wait(Math.max(1L, remainingNanos / 1_000_000L)); - } catch (InterruptedException ignored) { - interrupted = true; + quiescenceWaiters++; + try { + while (inService != null && inService.ring == ring) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0) { + return false; + } + try { + // Round up so a sub-millisecond remainder still waits + // instead of spinning through wait(0) == wait-forever. + lock.wait(Math.max(1L, remainingNanos / 1_000_000L)); + } catch (InterruptedException ignored) { + interrupted = true; + } } + } finally { + quiescenceWaiters--; } } return true; @@ -553,7 +566,14 @@ private void serviceRing(RingEntry e) { } finally { synchronized (lock) { inService = null; - lock.notifyAll(); + // Wake quiescence barriers only when one is actually parked. + // In production no engine ever waits here (owned-manager + // close joins the worker thread instead), so the per-tick + // pass must not pay an unconditional notifyAll. notifyAll, + // not notify: distinct waiters may await different rings. + if (quiescenceWaiters > 0) { + lock.notifyAll(); + } } } } From 930175fce064feaa8c98b4d915dc4386696a3ffc Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 19:19:57 +0100 Subject: [PATCH 06/26] fix(qwp): transfer slot cleanup to the worker's exit path and recover retired pool slots When an owned manager's bounded join timed out during engine close, the engine leaked ring + watermark mmaps and the slot flock until process exit, the sender latched slotLockReleased=false forever, and SenderPool retired the slot permanently (leakedSlots++) - a transiently-slow SF filesystem op at close time (> workerJoinTimeoutMillis, default 5 s) permanently ratcheted pool capacity down, even though the worker often exits moments later. - new SegmentManager.deferUntilWorkerExit(cleanup): hands an action to the worker-loop exit block, which runs strictly after the final service pass - the last point the worker can touch any slot path. Registration and the exit block's workerLoopExited flip share the manager lock, so the handoff is exactly-once: accepted while the worker is live, rejected (caller cleans up inline) once it exited - CursorSendEngine.close(): on an owned-manager join timeout, terminal cleanup (ring, watermark, drained-file unlink, flock release) now transfers to the worker's exit path instead of leaking; the quiescence gate is unchanged - the slot stays locked until the pass provably ends - exactly-once via a terminalCleanupClaimed CAS, deliberately not the engine monitor: a retried close() holds the monitor while joining the worker, and the worker cannot die until its cleanup returns - monitor-based exclusion would stall that close() for its full join budget. With the CAS the worker never blocks and the join returns as soon as the pass ends - QwpWebSocketSender.isSlotLockReleased() is now monotonic, not frozen: it re-probes the retained engine (volatile reads only, safe under the pool lock) and flips true once the deferred cleanup - worker exit path or delegated I/O-thread close - releases the flock - SenderPool re-probes retired slots (housekeeper reapIdle tick and capacity-starved borrows just before parking) and returns a recovered index to the free set: leakedSlots goes back down and a would-be borrow timeout becomes an immediate creation. A persistent non-zero leakedSlotCount() now means a genuinely wedged worker - shared-manager engines (test-only construction) keep the old leak-and-retry-close contract: their worker serves other rings and has no exit to defer to; startup-recovery retirements also stay permanent - regression: deferUntilWorkerExit contract test, owned-close handoff test (slot reacquirable after worker exit with NO close() retry), pool recovery via reapIdle and via capacity-starved borrow; the SlotLockReleasedContractTest leak-path pin updated to the new monotonic-getter contract --- .../qwp/client/QwpWebSocketSender.java | 88 +++++-- .../client/sf/cursor/CursorSendEngine.java | 241 ++++++++++++------ .../qwp/client/sf/cursor/SegmentManager.java | 60 +++++ .../io/questdb/client/impl/SenderPool.java | 77 +++++- .../client/SlotLockReleasedContractTest.java | 20 +- ...CursorSendEngineSlotReacquisitionTest.java | 117 ++++++++- .../cursor/SegmentManagerCloseRaceTest.java | 92 +++++++ .../client/test/impl/SenderPoolSfTest.java | 113 ++++++++ 8 files changed, 697 insertions(+), 111 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index c0998bcf..f1e3533c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -281,8 +281,17 @@ public class QwpWebSocketSender implements Sender { // teardown path). Stays false if an I/O or manager worker did not stop and // cursorEngine retained the flock, so the owning pool MUST keep the slot // reserved rather than hand the still-locked dir to the next borrow - // ("sf slot already in use"). - private boolean slotLockReleased; + // ("sf slot already in use"). May flip to true LATER via the getter's + // re-probe of retainedEngine, once the deferred cleanup (manager-worker + // exit path or delegated I/O-thread close) releases the flock — pools + // re-probe retired slots to recover their capacity. volatile: written on + // the closing thread, read by pool threads. + private volatile boolean slotLockReleased; + // Engine whose close() could not complete during sender close() — its + // cleanup is pending on a worker/I/O-thread exit path. isSlotLockReleased() + // re-probes it so a late flock release becomes visible to the owning pool. + // Only ever set inside close(); null for a sender that closed cleanly. + private volatile CursorSendEngine retainedEngine; private int pendingRowCount; private SenderProgressDispatcher progressDispatcher; // Async-delivery sink for ack-watermark advances. Default no-op; a @@ -1190,24 +1199,33 @@ public void close() { // close actually runs, so the pool must not reuse the slot // meanwhile. A false return means the thread exited between // the failed close() and now — then closing here is safe. - if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null - && !cursorSendLoop.delegateEngineClose()) { - CursorSendEngine engine = cursorEngine; - try { - engine.close(); - } catch (Throwable t) { - LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - if (engine.isCloseCompleted()) { - cursorEngine = null; - ownsCursorEngine = false; - slotLockReleased = true; + if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null) { + if (cursorSendLoop.delegateEngineClose()) { + // The I/O thread adopted the engine close and runs it on + // its exit path. Keep the engine visible for re-probing: + // isSlotLockReleased() flips true once that close + // completes, letting a pool recover the retired slot. + retainedEngine = cursorEngine; } else { - // Preserve the engine and report the retained flock. - // Sender.close() is idempotent, so this is a deliberate - // safe leak until process exit rather than a retry path. - slotLockReleased = false; + CursorSendEngine engine = cursorEngine; + try { + engine.close(); + } catch (Throwable t) { + LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + if (engine.isCloseCompleted()) { + cursorEngine = null; + ownsCursorEngine = false; + slotLockReleased = true; + } else { + // Engine cleanup is pending on its manager worker's + // exit path (or leaked, for a shared manager). Report + // the retained flock now; the getter re-probes the + // retained engine so a late release is not lost. + slotLockReleased = false; + retainedEngine = engine; + } } } rethrowTerminal(terminalError); @@ -1257,7 +1275,12 @@ public void close() { // The manager worker did not quiesce. Preserve ownership // and report the retained flock so pools retire this slot. // Repeated Sender.close() calls remain no-ops by contract. + // Engine cleanup was handed to the worker's exit path + // (owned manager); the getter re-probes the retained + // engine so the pool can reclaim the slot once cleanup + // actually completes. slotLockReleased = false; + retainedEngine = engine; } } else { // This sender owns no cursor engine holding an SF flock. @@ -1306,13 +1329,30 @@ public void close() { } /** - * True once {@link #close()} has released the store-and-forward slot - * flock. False means an I/O or manager worker did not stop and close() - * retained the lock and worker-reachable resources; the owning pool must - * keep the slot index reserved instead of reusing the still-locked dir. + * True once the store-and-forward slot flock has been released. False + * means an I/O or manager worker did not stop and close() retained the + * lock and worker-reachable resources; the owning pool must keep the slot + * index reserved instead of reusing the still-locked dir. + *

    + * Not a one-shot snapshot: when close() left engine cleanup pending on a + * worker/I/O-thread exit path, this re-probes the retained engine and + * latches true the moment that cleanup completes — pools re-probe retired + * slots through this getter to recover their capacity. Monotonic: + * false→true only, never back. Lock-free (volatile reads) so pools may + * call it under their capacity lock. */ public boolean isSlotLockReleased() { - return slotLockReleased; + if (slotLockReleased) { + return true; + } + CursorSendEngine engine = retainedEngine; + if (engine != null && engine.isCloseCompleted()) { + // Benign latch race: concurrent callers may both observe the + // completed cleanup and both write true. + slotLockReleased = true; + return true; + } + return false; } @Override diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 54f675f9..9cd52169 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -29,6 +29,7 @@ import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; /** @@ -119,7 +120,20 @@ public final class CursorSendEngine implements QuietCloseable { // leak the ring/watermark/slot lock — in that case a later close() call // retries the cleanup (the worker may have exited by then). Guarded by // the synchronized close() method; never read elsewhere. - private boolean closeCompleted; + // volatile: latched by finishClose(), but read lock-free by + // isCloseCompleted() from pool threads re-probing a retired slot (see the + // getter for why it must not synchronize). + private volatile boolean closeCompleted; + // Exactly-once claim on the terminal cleanup (finishClose). Contended by + // close() and a worker-exit handoff (completeDeferredClose); whoever wins + // the CAS runs the cleanup, the loser never touches ring/watermark/flock. + // Deliberately NOT the engine monitor: a retried close() holds the + // monitor while joining the manager worker, and the worker cannot die + // until its exit-path cleanup finishes — monitor-based exclusion would + // stall that close() for its full join budget (test-visible livelock). + // With the CAS the worker's cleanup never blocks, so the join returns as + // soon as the pass ends. + private final AtomicBoolean terminalCleanupClaimed = new AtomicBoolean(); // Producer-thread-only: timestamp of the last "we're backpressured" log // line, used to throttle. Plain long is fine. private long lastBackpressureLogNs; @@ -497,79 +511,136 @@ public synchronized void close() { // against potentially-fresh server state — duplicate writes when // the server has no dedup state for those messageSequences. // Memory mode has no files to unlink. - // The whole close sequence runs under try/finally so the slot lock - // is released whenever it is safe to do so, even if manager/ring - // close or unlink throws — otherwise a kernel-held flock outlives - // the engine and the next sender for the same slot collides on a - // lock the dead engine never released. The one deliberate exception - // is a manager worker that failed to quiesce: releasing the lock - // then would let a replacement engine acquire the slot while the - // stale worker can still create/unlink segment paths inside it. - boolean workerQuiescent = false; + // + // Cleanup is gated on worker quiescence: releasing the ring, + // watermark, segment files or the slot lock while the manager worker + // is still mid service pass would let a replacement engine acquire + // the slot and have its files unlinked by the stale worker — + // store-and-forward data loss after restart. When the bounded worker + // join times out on an owned manager, cleanup OWNERSHIP TRANSFERS to + // the worker's exit path (deferUntilWorkerExit): the worker is + // provably the last thread able to touch the slot directory, so it + // releases everything the moment its in-flight pass finishes instead + // of leaking the slot until process exit. + // + // "Fully drained" includes BOTH the obvious case (every published + // FSN has been acked) AND the never-published case (publishedFsn + // < 0). The latter matters because a drainer adopting an empty + // orphan slot — segments filtered as empty by recovery, engine + // recreates a fresh sf-initial.sfa — would otherwise leave that + // fresh empty file behind, the next scanner finds it, adopts the + // slot again, and the cycle repeats forever (M6). + // Own try/catch so sabotaged/broken ring state cannot skip the + // quiescence barrier below or the slot-lock release in finishClose. + boolean drained = false; + try { + drained = sfDir != null + && (ring.publishedFsn() < 0 + || ring.ackedFsn() >= ring.publishedFsn()); + } catch (Throwable ignored) { + } + final boolean fullyDrained = drained; + // Each cleanup step in its own try/catch so a single failure + // doesn't strand later cleanups — mirrors the constructor's + // catch block. Without this, a throw from manager.deregister + // or manager.close() would leave the ring mmap'd and any + // residual .sfa files on disk, where the next sender can + // adopt them and replay already-acked data. try { - // "Fully drained" includes BOTH the obvious case (every published - // FSN has been acked) AND the never-published case (publishedFsn - // < 0). The latter matters because a drainer adopting an empty - // orphan slot — segments filtered as empty by recovery, engine - // recreates a fresh sf-initial.sfa — would otherwise leave that - // fresh empty file behind, the next scanner finds it, adopts the - // slot again, and the cycle repeats forever (M6). - // Own try/catch so sabotaged/broken ring state cannot skip the - // quiescence barrier below or the slot-lock release in finally. - boolean fullyDrained = false; + manager.deregister(ring); + } catch (Throwable ignored) { + } + // Quiescence barrier. deregister alone only removes the entry + // from the registry — the worker may still be mid service pass + // for this ring (creating a spare file, trimming, unlinking). + boolean workerQuiescent = false; + if (ownsManager) { + // Stopping and reaping a private manager is a stronger barrier + // than waiting for this ring alone. Do it directly so a stuck + // worker consumes at most one workerJoinTimeoutMillis budget, + // rather than one here and a second one in manager.close(). try { - fullyDrained = sfDir != null - && (ring.publishedFsn() < 0 - || ring.ackedFsn() >= ring.publishedFsn()); + manager.close(); } catch (Throwable ignored) { } - // Each cleanup step in its own try/catch so a single failure - // doesn't strand later cleanups — mirrors the constructor's - // catch block. Without this, a throw from manager.deregister - // or manager.close() would leave the ring mmap'd and any - // residual .sfa files on disk, where the next sender can - // adopt them and replay already-acked data. try { - manager.deregister(ring); + workerQuiescent = manager.isWorkerReaped(); } catch (Throwable ignored) { } - // Quiescence barrier. deregister alone only removes the entry - // from the registry — the worker may still be mid service pass - // for this ring (creating a spare file, trimming, unlinking). - // Releasing the ring, watermark, segment files, or the slot lock - // while that pass is in flight lets the stale worker unlink a - // path that a replacement engine — which can acquire the slot - // the moment the lock is released — is actively writing through: - // store-and-forward data loss after restart. - if (ownsManager) { - // Stopping and reaping a private manager is a stronger barrier - // than waiting for this ring alone. Do it directly so a stuck - // worker consumes at most one workerJoinTimeoutMillis budget, - // rather than one here and a second one in manager.close(). - try { - manager.close(); - } catch (Throwable ignored) { - } - try { - workerQuiescent = manager.isWorkerReaped(); - } catch (Throwable ignored) { - } - } else { - // A shared manager must keep serving its other rings, so wait - // only for the deregistered ring's current pass to finish. - try { - workerQuiescent = manager.awaitRingQuiescence(ring); - } catch (Throwable ignored) { - } + } else { + // A shared manager must keep serving its other rings, so wait + // only for the deregistered ring's current pass to finish. + try { + workerQuiescent = manager.awaitRingQuiescence(ring); + } catch (Throwable ignored) { + } + } + if (!workerQuiescent && ownsManager) { + // Ownership handoff: manager.close() already stopped the worker + // loop (running=false), so the worker exits as soon as its + // in-flight pass finishes — it is merely slow, or wedged in a + // syscall. Either way it is the last thread able to touch the + // slot, so hand it the terminal cleanup instead of leaking the + // slot until process exit. completeDeferredClose and a retried + // close() race through the terminalCleanupClaimed CAS, so the + // cleanup runs exactly once and the worker never blocks on the + // engine monitor a retried close() holds while joining it. + boolean handedOff = false; + try { + handedOff = manager.deferUntilWorkerExit(() -> completeDeferredClose(fullyDrained)); + } catch (Throwable ignored) { } - if (!workerQuiescent) { - LOG.error("SF manager worker did not quiesce during engine close; leaking the " - + "ring, watermark and slot lock so a stale worker cannot corrupt a " - + "future engine on slot {}. The kernel releases the slot flock on " - + "process exit; close() may be invoked again to retry cleanup once " - + "the worker has exited.", sfDir == null ? "" : sfDir); + if (handedOff) { + LOG.error("SF manager worker did not quiesce during engine close; ring, watermark " + + "and slot-lock release are handed to the worker's exit path and run the " + + "moment its in-flight service pass finishes. The slot stays locked (and " + + "isCloseCompleted() false) until then, so no replacement engine can race " + + "the stale worker on slot {}", sfDir == null ? "" : sfDir); return; } + // Handoff rejected: the worker loop exited between the failed + // bounded join and the registration attempt (both sides share the + // manager's lock, so this observation is exact). A worker past + // its loop can never touch slot paths again — inline cleanup is + // as safe as a reaped worker. + workerQuiescent = true; + } + if (!workerQuiescent) { + // Shared (caller-owned) manager: its worker keeps serving other + // rings indefinitely, so there is no exit path to hand cleanup + // to — leak and let a retried close() reclaim once the pass ends. + LOG.error("SF manager worker did not quiesce during engine close; leaking the " + + "ring, watermark and slot lock so a stale worker cannot corrupt a " + + "future engine on slot {}. The kernel releases the slot flock on " + + "process exit; close() may be invoked again to retry cleanup once " + + "the worker has exited.", sfDir == null ? "" : sfDir); + return; + } + if (!terminalCleanupClaimed.compareAndSet(false, true)) { + // A worker-exit handoff from an earlier close() attempt owns the + // terminal cleanup: it has run or is finishing right now on the + // exiting worker. closeCompleted flips the moment it is done — + // owners observe it via isCloseCompleted(), never by re-running + // the cleanup here (that would double-release ring/flock). + return; + } + finishClose(fullyDrained); + } + + /** + * Terminal cleanup: closes the ring and watermark, unlinks drained + * segment files, releases the slot flock, and latches + * {@link #closeCompleted}. The caller must hold the engine monitor and + * must have established that the manager worker can no longer touch the + * slot directory (reaped, provably exited, or running this on its own + * exit path) AND have won the {@link #terminalCleanupClaimed} CAS — the + * claim token, not the engine monitor, is the exclusion between a + * worker-exit handoff and a retried close(), so ring/watermark/flock are + * never double-released and the worker's cleanup can never deadlock + * against a close() that holds the monitor while joining the worker. + */ + private void finishClose(boolean fullyDrained) { + try { try { ring.close(); } catch (Throwable ignored) { @@ -600,11 +671,10 @@ public synchronized void close() { } closeCompleted = true; } finally { - // Gate on quiescence: releasing the flock while a stale worker - // can still touch this slot directory hands the slot to a new - // engine that the worker may then corrupt. Leaking the fd is - // the safe failure mode — the kernel drops it on process exit. - if (workerQuiescent && slotLock != null) { + // Reaching finishClose at all requires established quiescence, so + // releasing the flock is safe even if a step above threw. Leaking + // it would strand the slot until process exit for no reason. + if (slotLock != null) { try { slotLock.close(); } catch (Throwable ignored) { @@ -614,13 +684,42 @@ public synchronized void close() { } } + /** + * Runs on the manager worker's exit path when {@link #close()} handed + * cleanup ownership to the worker (see {@code deferUntilWorkerExit}). + * Deliberately does NOT take the engine monitor: a retried close() can + * hold it while joining this very worker, and the thread cannot die until + * this method returns — the {@link #terminalCleanupClaimed} CAS provides + * the exactly-once exclusion instead, so the worker always exits promptly + * and the racing close() converges via {@code isWorkerReaped()}. + */ + private void completeDeferredClose(boolean fullyDrained) { + if (!terminalCleanupClaimed.compareAndSet(false, true)) { + // A retried close() (or an earlier duplicate handoff) already ran + // the terminal cleanup. + return; + } + finishClose(fullyDrained); + LOG.info("deferred SF engine cleanup completed on manager-worker exit; slot released: {}", + sfDir == null ? "" : sfDir); + } + /** * Whether {@link #close()} completed all cleanup, including releasing the * SF slot lock. A false value after close means manager-worker quiescence * could not be confirmed and the worker-reachable resources were retained - * deliberately. Owners must not report or reuse the slot in that case. + * deliberately — either handed to the worker's exit path (owned manager), + * which flips this to true the moment the worker's in-flight pass + * finishes, or leaked until a retried close() (shared manager). Owners + * must not reuse the slot while this is false; pools may re-probe it to + * recover a retired slot's capacity once it flips. + *

    + * Deliberately unsynchronized ({@code closeCompleted} is volatile): pools + * probe this under their own capacity lock, and the deferred cleanup can + * hold the engine monitor through munmap/unlink syscalls — a synchronized + * getter would stall the pool's hot borrow path behind them. */ - public synchronized boolean isCloseCompleted() { + public boolean isCloseCompleted() { return closeCompleted; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 726374d6..5deaf2ca 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -109,6 +109,14 @@ public final class SegmentManager implements QuietCloseable { // {@link #lock}; cleared with lock.notifyAll() so awaitRingQuiescence can // block until an in-flight pass for a just-deregistered ring finishes. private RingEntry inService; + // Cleanup actions handed to the worker's exit block by an owning engine + // whose close() found the worker still mid service pass after the bounded + // join timed out (see deferUntilWorkerExit). Guarded by {@link #lock}; + // consumed exactly once by the worker-loop finally, which runs them + // OUTSIDE `lock`: they perform syscalls (munmap/unlink/flock release), + // and no caller-facing lock may ever be held while running third-party + // cleanup code. + private ObjList exitCleanups; // Number of threads currently parked in awaitRingQuiescence()'s wait // loop. Guarded by {@link #lock}. The worker's per-pass finally block // only calls lock.notifyAll() when this is > 0, so the steady-state @@ -258,6 +266,37 @@ public synchronized void close() { } } + /** + * Hands a cleanup action to the worker thread's exit block, to run + * strictly after the worker loop has finished its final service pass -- + * i.e. after the last point where the worker can create, write or unlink + * anything under a registered ring's slot directory. Returns {@code false} + * when the worker loop has already exited (or the worker never started): + * the caller must run the cleanup itself, which is equally safe for the + * same reason -- no further worker access to the slot is possible. + *

    + * This is the slot-ownership transfer used by an owning engine's close() + * when the bounded worker join timed out: instead of retiring the slot + * until process exit, ring/watermark/flock release moves to the worker, + * which is provably the last thread able to touch the slot directory. + * The registration here and the exit block's {@code workerLoopExited} + * flip share {@link #lock}, so the cleanup runs exactly once: either it + * is registered before the flip and the worker runs it, or the flip won + * and this method rejects the handoff. + */ + public boolean deferUntilWorkerExit(Runnable cleanup) { + synchronized (lock) { + if (workerLoopExited || workerThread == null) { + return false; + } + if (exitCleanups == null) { + exitCleanups = new ObjList<>(); + } + exitCleanups.add(cleanup); + return true; + } + } + /** * Quiescence barrier for {@link #deregister(SegmentRing)}. Blocks until * the worker thread is provably no longer executing a service pass for @@ -765,12 +804,33 @@ private void workerLoop() { // pathScratch ownership to this thread (see close()). Freeing it // here reclaims the native buffer even when the worker outlives // every close() attempt — nobody else retries manager cleanup. + ObjList cleanups; synchronized (lock) { workerLoopExited = true; if (scratchHandedToWorker && !scratchFreed) { scratchFreed = true; pathScratch.close(); } + cleanups = exitCleanups; + exitCleanups = null; + } + // Deferred engine cleanups (see deferUntilWorkerExit) run OUTSIDE + // `lock`: they perform syscalls (munmap, unlink, flock release) + // and must never execute under a lock that close()/register/ + // deregister callers contend on. Running them after the loop body + // is what makes the handoff safe: this thread can no longer touch + // any slot path. They must also never block on a caller-held + // monitor — a retried engine.close() joins this thread while + // holding the engine monitor, which is why the engine side uses a + // lock-free claim (CAS), not synchronization, for exactly-once. + if (cleanups != null) { + for (int i = 0, n = cleanups.size(); i < n; i++) { + try { + cleanups.getQuick(i).run(); + } catch (Throwable t) { + LOG.error("deferred engine cleanup failed on manager-worker exit", t); + } + } } } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 31e50f31..798a931a 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -171,11 +171,21 @@ public final class SenderPool implements AutoCloseable { // down on another thread. Guarded by lock. private int pendingLeaseTeardowns; // Slots whose delegate close() returned with the SF flock still held - // because an I/O or manager worker did not stop. Permanently consumed: + // because an I/O or manager worker did not stop. Consumed while retired: // never freed and never reused, so no borrow ever hands out a still- // locked slot dir. Counted in the cap check so the lost capacity is - // accounted for. Guarded by lock; only ever ticks for SF slots. + // accounted for. NOT necessarily permanent: engine cleanup may be pending + // on a worker/I/O-thread exit path, so reprobeRetiredSlots() re-checks + // retiredSlots and returns any index whose flock has since dropped. + // Guarded by lock; only ever ticks for SF slots. private int leakedSlots; + // The retired slots behind the leakedSlots count (runtime reclaim paths + // only; startup-recovery retirements stay permanent — their transient + // recoverer sender is out of scope by retire time). Re-probed by + // reprobeRetiredSlots() so a late flock release (deferred engine cleanup + // on a worker exit path) restores the pool's capacity instead of + // ratcheting it down until process exit. Guarded by lock. + private final ArrayList retiredSlots = new ArrayList<>(); // SF slots currently held by the in-range startup-recovery pass // (recoverOneSlotStep): each is reserved under `lock` for the // duration of its drain and counted in the borrow() cap check so a @@ -773,6 +783,12 @@ public PooledSender borrow() { throw new LineSenderException( "timed out waiting for a Sender from the pool after " + acquireTimeoutMillis + "ms"); } + // Capacity-starved: before parking, re-probe retired slots — a + // deferred engine cleanup may have released a flock since the + // retire, and the freed index can admit a creation right now. + if (reprobeRetiredSlots()) { + continue; + } try { remainingNanos = slotReleased.awaitNanos(remainingNanos); } catch (InterruptedException e) { @@ -1029,6 +1045,11 @@ public void reapIdle() { if (closed) { return; } + // Housekeeper tick doubles as the retired-slot recovery driver: + // a slot retired because its worker did not stop is re-probed + // here and returns to the free set once the deferred cleanup + // finally released its flock. + reprobeRetiredSlots(); Iterator it = available.iterator(); while (it.hasNext() && all.size() > minSize) { SenderSlot s = it.next(); @@ -1107,12 +1128,15 @@ public int totalSize() { } /** - * Snapshot of the number of SF slots permanently retired because a + * Snapshot of the number of SF slots currently retired because a * delegate {@code close()} returned with the slot flock still held after - * an I/O or manager worker did not stop. Each leaked slot permanently lowers the - * pool's effective capacity ({@code maxSize - leakedSlotCount()}). A - * non-zero, growing value explains a pool that has started timing out - * every {@code borrow()}. For metrics and tests. + * an I/O or manager worker did not stop. Each leaked slot lowers the + * pool's effective capacity ({@code maxSize - leakedSlotCount()}) while + * retired. Retired slots are re-probed (housekeeper tick and + * capacity-starved borrows) and recovered once the delegate's deferred + * cleanup releases the flock, so the count can go back down; a non-zero, + * persistent value means a worker is still wedged and explains a pool + * that has started timing out {@code borrow()}. For metrics and tests. */ public int leakedSlotCount() { lock.lock(); @@ -1304,9 +1328,44 @@ private boolean reclaimSlot(SenderSlot s, String context) { return true; } leakedSlots++; - LOG.warn("SF slot {} retired permanently{}: delegate close() returned with the flock still held " + - "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable [leakedSlots={}]", + retiredSlots.add(s); + LOG.warn("SF slot {} retired{}: delegate close() returned with the flock still held " + + "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable " + + "[leakedSlots={}]; the slot is re-probed and recovered if the worker releases the flock later", s.slotIndex(), context, maxSize - leakedSlots, maxSize, leakedSlots); return false; } + + /** + * Re-probes every retired slot (see {@link #reclaimSlot}) and returns to + * the free set any whose delegate now reports the flock released — the + * deferred engine cleanup (manager-worker or I/O-thread exit path) has + * run since the retire. Restores {@code leakedSlots} capacity and signals + * waiters so a parked borrow can admit a creation immediately. + *

    + * Caller must hold {@code lock}. The probe is lock-free on the delegate + * side ({@code isSlotLockReleased()} reads volatiles only), so holding + * the pool lock across it cannot stall behind delegate teardown. + * + * @return {@code true} if at least one slot's capacity was recovered + */ + private boolean reprobeRetiredSlots() { + boolean recovered = false; + for (int i = retiredSlots.size() - 1; i >= 0; i--) { + SenderSlot s = retiredSlots.get(i); + if (flockReleased(s)) { + retiredSlots.remove(i); + leakedSlots--; + freeSlotIndex(s.slotIndex()); + recovered = true; + LOG.info("SF slot {} recovered: deferred cleanup released the flock after retirement; " + + "pool capacity restored, now {} of {} usable [leakedSlots={}]", + s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); + } + } + if (recovered) { + slotReleased.signalAll(); + } + return recovered; + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index 97ae9dbe..e8ef2670 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -58,6 +58,10 @@ *

  • Leak path — a {@code close()} that retains the lock because an * I/O or manager worker did not stop reports * {@code isSlotLockReleased() == false}.
    • + *
  • Recovery path — the getter is monotonic but not frozen: it + * re-probes the retained engine, so once the deferred cleanup (worker + * exit path or a retried close) releases the flock, the getter flips to + * {@code true} and a pool that retired the slot can recover it.
    • * * The pool's {@code flockReleased(s)} treats {@code false} as "flock still held, * retire the slot permanently". Before this test that contract was driven only @@ -191,7 +195,9 @@ public void testSlotLockNotReleasedWhenIoThreadRefusesToStop() throws Exception * Manager-worker leak path: engine close retains the slot while a shared * manager is still inside a service pass. The sender must expose that * retained flock to SenderPool. Repeated sender close calls remain no-ops; - * the test cleans the deliberately retained engine up directly. + * the test cleans the deliberately retained engine up directly — and once + * that cleanup completes, the sender must expose the released flock so a + * pool that retired the slot can recover its capacity. */ @Test(timeout = 30_000L) public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception { @@ -254,13 +260,17 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception releaseWorker.countDown(); manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); - // Test-only cleanup through the retained local reference. The - // sender conservatively keeps reporting false, as required by - // a pool that may already have retired this slot. + // Test-only cleanup through the retained local reference (the + // shared-manager path has no worker-exit handoff to defer to; + // a retried close() is its reclaim driver). engine.close(); Assert.assertTrue("direct cleanup did not complete after worker exit", engine.isCloseCompleted()); - Assert.assertFalse("sender must not revise the result after close returned", + // Recovery contract: the sender re-probes its retained engine, + // so the completed cleanup (and released flock) MUST become + // visible — this is what lets SenderPool recover a slot it + // had already retired as leaked. + Assert.assertTrue("sender must expose the late flock release to the pool", wss.isSlotLockReleased()); try (SlotLock probe = SlotLock.acquire(slot)) { Assert.assertNotNull("slot must be acquirable after direct cleanup", probe); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 9afb86eb..892bdc59 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -59,8 +59,10 @@ * ({@link SegmentManager#awaitRingQuiescence}) after {@code deregister} and * refuse to release any worker-reachable resource (ring, watermark, segment * files, slot lock) until the barrier confirms the worker cannot touch the - * slot again. On barrier timeout the engine deliberately leaks and a later - * {@code close()} retries the cleanup. + * slot again. On barrier timeout an owned-manager engine hands cleanup + * ownership to the worker's exit path (see + * {@code SegmentManager.deferUntilWorkerExit}); a shared-manager engine + * deliberately leaks and a later {@code close()} retries the cleanup. */ public class CursorSendEngineSlotReacquisitionTest { @@ -296,6 +298,117 @@ public void testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass() throws }); } + /** + * The ownership handoff (owned manager): when close() cannot confirm + * worker quiescence within the bounded join, the terminal cleanup (ring, + * watermark, flock release) transfers to the worker's exit path — the + * worker is provably the last thread able to touch the slot directory. + * Once the parked pass is released the worker must run the cleanup + * itself, WITHOUT any retried {@code close()}: {@code isCloseCompleted()} + * flips true and the slot becomes acquirable again. This is what lets a + * pool recover a retired slot instead of losing its capacity until + * process exit. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseHandsCleanupToWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-handoff-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass (see + // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass). + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: fill the active segment and rotate onto the spare; + // the worker's next tick re-enters the install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: owned close with the worker provably mid-pass. The + // bounded join times out; cleanup ownership is handed to the + // worker's exit path. Every worker-reachable resource — above + // all the slot flock — must still be retained at this point. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete while the worker holds the handoff", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock while its manager worker " + + "was still mid service pass — the handoff must not weaken the " + + "quiescence gate"); + } catch (Exception expected) { + // good — slot retained while the worker can still touch it. + } + + // Phase 4 — the contract under test: release the worker and do + // NOT retry close(). The worker finishes its pass, exits, and + // runs the deferred cleanup itself, flipping isCloseCompleted + // and releasing the flock with no further caller action. + releaseWorker.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError( + "deferred cleanup never ran on manager-worker exit — the slot " + + "would stay retired until process exit"); + } + Thread.sleep(1); + } + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the worker-exit cleanup", probe); + } catch (Exception e) { + throw new AssertionError("worker-exit cleanup did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + /** * An engine that owns its manager must use the whole-manager stop/join as * its only quiescence barrier. Calling the per-ring barrier first would diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index 4e62f8ca..9f6fc225 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -204,6 +204,98 @@ public void testCloseDoesNotFreePathScratchWhenWorkerStillAlive() throws Excepti }); } + /** + * Pins the {@link SegmentManager#deferUntilWorkerExit} handoff contract + * that {@code CursorSendEngine.close()}'s slot-ownership transfer depends + * on: + *
      + *
    • rejects the handoff ({@code false}) when no worker ever started — + * the caller must clean up inline;
      • + *
    • accepts it ({@code true}) while the worker is live-but-slow + * mid service pass after a timed-out close(), and runs the cleanup + * exactly when the worker exits — never while the pass is still in + * flight;
      • + *
    • rejects it again once the worker loop has exited/been reaped.
      • + *
    + */ + @Test(timeout = 15_000L) + public void testDeferUntilWorkerExitRunsCleanupAfterFinalPass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/defer-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + CountDownLatch cleanupRan = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + try { + // Never-started manager: no worker will ever run the cleanup, + // and none can touch a slot — the caller must do it inline. + Assert.assertFalse("never-started manager must reject the handoff", + manager.deferUntilWorkerExit(cleanupRan::countDown)); + + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Timed-out close: running=false, worker parked mid-pass — the + // exact state CursorSendEngine.close() hands ownership over in. + manager.setWorkerJoinTimeoutMillis(50L); + manager.close(); + Assert.assertFalse("worker must not be reaped while parked mid-pass", + manager.isWorkerReaped()); + + Assert.assertTrue("live-but-slow worker must accept the handoff", + manager.deferUntilWorkerExit(cleanupRan::countDown)); + Assert.assertEquals("cleanup must not run while the pass is still in flight", + 1, cleanupRan.getCount()); + + // Release the pass; the worker loop observes running=false, + // exits, and must run the deferred cleanup on its way out. + releaseWorker.countDown(); + Assert.assertTrue("cleanup never ran on worker exit", + cleanupRan.await(10, TimeUnit.SECONDS)); + + // Reap the exited worker, then: no live worker, no handoff. + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + manager.close(); + managerClosed = true; + Assert.assertFalse("reaped manager must reject the handoff", + manager.deferUntilWorkerExit(() -> { + })); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + Thread.interrupted(); + manager.close(); + } + ring.close(); + } + }); + } + /** * Pins the {@link SegmentManager#awaitRingQuiescence} contract that * {@code CursorSendEngine.close()} depends on: diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index cfef2feb..75aca494 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -658,6 +658,119 @@ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlockDuringReap() throw }); } + @Test + public void testRetiredSlotRecoveredByHousekeeperAfterLateFlockRelease() throws Exception { + // Recovery twin of testSlotLeakedWhenDelegateCloseDoesNotReleaseFlock: + // a slot retired because close() returned with the flock still held is + // NOT lost until process exit. Engine cleanup may complete later on a + // worker/I/O-thread exit path (isSlotLockReleased() re-probes the + // retained engine), and the housekeeper's reapIdle() tick must then + // return the index to the free set: leakedSlots back down, slotInUse + // cleared, full capacity restored. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + // Forge the retire: real teardown first (no native leaks), + // then clear slotLockReleased so discardBroken retires the + // slot as leaked. + Sender delegate = getDelegate(a); + delegate.close(); + setBooleanField(delegate, "slotLockReleased", false); + invokeDiscardBroken(pool, a); + Assert.assertEquals("precondition: one slot must be retired", + 1, pool.leakedSlotCount()); + + // Forge the late release: the deferred cleanup finished and + // the delegate now reports the flock dropped (in production + // this flip comes from isSlotLockReleased() re-probing the + // retained engine after the manager worker exited). + setBooleanField(delegate, "slotLockReleased", true); + + // The housekeeper tick is the recovery driver. + pool.reapIdle(); + + Assert.assertEquals("recovered slot must leave the leaked count", + 0, pool.leakedSlotCount()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + Assert.assertFalse("recovered slot index 0 must return to the free set", + slotInUse[0]); + + // Full capacity restored: with maxSize=2, two concurrent + // borrows must succeed again (index 0 is reusable — its + // flock is genuinely free). + PooledSender b = pool.borrow(); + PooledSender c = pool.borrow(); + try { + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + b.close(); + } + } + } + }); + } + + @Test + public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { + // Borrow-path twin of the housekeeper recovery test: a borrow that + // would otherwise park on the cap check must re-probe retired slots + // before waiting, so a late flock release converts a guaranteed + // borrow timeout into an immediate creation on the recovered index — + // no housekeeper tick required. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + Sender delegate = getDelegate(a); + delegate.close(); + setBooleanField(delegate, "slotLockReleased", false); + invokeDiscardBroken(pool, a); + Assert.assertEquals("precondition: one slot must be retired", + 1, pool.leakedSlotCount()); + + // One live borrow + one retired slot = cap reached. + PooledSender b = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-1"))); + try { + // Late release lands while the pool is capacity-starved. + setBooleanField(delegate, "slotLockReleased", true); + + // The next borrow hits the cap check, re-probes, frees + // index 0, and must create on it instead of timing out. + PooledSender c = pool.borrow(); + try { + Assert.assertEquals("borrow must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + } + } finally { + b.close(); + } + } + } + }); + } + // ---------------------------------------------------------------------- // Recovery: stable slot ids let a re-created pool re-adopt unacked data. // ---------------------------------------------------------------------- From eea7f89fb4a799e236f4e82efa2da757680939cb Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 21:09:56 +0100 Subject: [PATCH 07/26] test(qwp): pin slot retention when worker-exit handoff registration fails A throw from deferUntilWorkerExit (allocation failure while building the handoff) carries no worker-liveness information, so close() must retain every worker-reachable resource instead of running terminal cleanup inline under a possibly-live worker. Add a test seam that throws from the registration path while the worker is provably mid service pass and assert the slot flock, ring and watermark are retained, close stays incomplete, and a retried close() after worker exit converges and releases the slot. --- .../qwp/client/sf/cursor/SegmentManager.java | 22 ++++ ...CursorSendEngineSlotReacquisitionTest.java | 114 ++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 5deaf2ca..51bd204e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -97,6 +97,13 @@ public final class SegmentManager implements QuietCloseable { // production; owned-engine close tests use it to prove they take only the // stronger whole-manager join path, not two sequential timeout budgets. private volatile Runnable beforeRingQuiescenceAwaitHook; + // Test seam: runs at the top of deferUntilWorkerExit, before the + // worker-liveness check. Null in production; registration-failure tests + // throw from it to simulate an allocation failure (OOM building the + // cleanup lambda or growing exitCleanups) while the worker is still + // live. Callers must treat such a throw as "worker state unknown", + // never as the exact false return meaning the worker loop has exited. + private volatile Runnable beforeExitCleanupRegistrationHook; // Test seam: runs on the worker thread just before the trim block's // synchronized(lock) entry. Null in production; only // SegmentManagerTrimDeregisterRaceTest installs it, to deterministically @@ -283,8 +290,18 @@ public synchronized void close() { * flip share {@link #lock}, so the cleanup runs exactly once: either it * is registered before the flip and the worker runs it, or the flip won * and this method rejects the handoff. + *

    + * May throw on allocation failure (the lambda at the call site, the + * {@code ObjList}, or its growth). A throw carries NO liveness + * information: the worker was never observed, so the caller must treat + * it as "worker possibly still live" and retain resources — never as + * the exact {@code false} return above. */ public boolean deferUntilWorkerExit(Runnable cleanup) { + Runnable hook = beforeExitCleanupRegistrationHook; + if (hook != null) { + hook.run(); + } synchronized (lock) { if (workerLoopExited || workerThread == null) { return false; @@ -465,6 +482,11 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { wakeWorker(); } + @TestOnly + public void setBeforeExitCleanupRegistrationHook(Runnable hook) { + this.beforeExitCleanupRegistrationHook = hook; + } + @TestOnly public void setBeforeInstallSyncHook(Runnable hook) { this.beforeInstallSyncHook = hook; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 892bdc59..ad25ff78 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -409,6 +409,120 @@ public void testOwnedEngineCloseHandsCleanupToWorkerExit() throws Exception { }); } + /** + * Registration-failure twin of + * {@link #testOwnedEngineCloseHandsCleanupToWorkerExit}: when + * {@code deferUntilWorkerExit} itself throws (allocation failure while + * building the handoff), close() must NOT mistake the swallowed throw + * for "worker already exited" and run the terminal cleanup inline — the + * worker is provably still mid service pass, so releasing the ring, + * watermark or slot flock here is the original stale-worker UAF/data-loss + * hazard. Every worker-reachable resource must be retained and the close + * must stay incomplete; a retried close() after the worker exits + * converges and releases the slot. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseRetainsSlotWhenHandoffRegistrationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-regfail-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass (see + // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass). + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: fill the active segment and rotate onto the spare; + // the worker's next tick re-enters the install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: make the handoff registration throw — simulating + // an OutOfMemoryError allocating the cleanup lambda/list — + // with the worker provably still mid service pass. close() + // must retain everything: no inline finishClose, no flock + // release, closeCompleted stays false. + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated allocation failure registering exit cleanup"); + }); + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete when handoff registration fails", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock after a failed handoff " + + "registration while the manager worker was still mid service " + + "pass — the swallowed throw was mistaken for proof the worker " + + "exited (stale-worker UAF/data-loss hazard)"); + } catch (Exception expected) { + // good — slot retained while the worker can still touch it. + } + + // Phase 4: clear the fault, release the worker (its loop was + // already stopped by the close attempt, so it exits), and + // retry close(). The retry must converge via isWorkerReaped() + // and release the slot. + manager.setBeforeExitCleanupRegistrationHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + engine.close(); + Assert.assertTrue("retried close must report complete cleanup", + engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the retried close", probe); + } catch (Exception e) { + throw new AssertionError("retried close() did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + manager.setBeforeExitCleanupRegistrationHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + /** * An engine that owns its manager must use the whole-manager stop/join as * its only quiescence barrier. Calling the per-ring barrier first would From 940e1307b061fce56e41dc2016cedd444dd8f9fe Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 21:10:08 +0100 Subject: [PATCH 08/26] fix(qwp): publish close completion only after confirmed slot flock release finishClose() wrote closeCompleted=true before slotLock.close(), so a pool thread could observe completion (isSlotLockReleased -> reprobeRetiredSlots), free the slot index, and admit a replacement sender whose SlotLock.acquire collided with the still-open flock fd -- a spurious "sf slot already in use" naming the process's own pid. SlotLock.close() also discarded the Files.close() result, reporting an unconfirmed release as completion. Reorder the terminal cleanup: release the flock first via the new SlotLock.release() (checks the close rc, retains the fd on failure so the lock state is never misreported), and publish closeCompleted only on a confirmed release. An unconfirmed release keeps closeCompleted false, degrading into the existing retired/leaked-slot accounting with the kernel's process-exit backstop. Add a @TestOnly hook between cleanup and release so the otherwise microsecond-wide window is deterministically testable; the new test parks the closer inside it and asserts completion is not observable while the flock is provably still held, then latches once released. --- .../client/sf/cursor/CursorSendEngine.java | 118 ++++++++++-- .../qwp/client/sf/cursor/SlotLock.java | 42 ++++- ...gineClosePublishAfterFlockReleaseTest.java | 176 ++++++++++++++++++ .../qwp/client/sf/cursor/SlotLockTest.java | 19 ++ 4 files changed, 334 insertions(+), 21 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 9cd52169..c9b85aea 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -28,6 +28,7 @@ import io.questdb.client.std.Files; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; @@ -115,15 +116,28 @@ public final class CursorSendEngine implements QuietCloseable { // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. private volatile boolean closed; - // True once close() has run its full cleanup sequence. Stays false when - // a close attempt could not confirm manager-worker quiescence and had to - // leak the ring/watermark/slot lock — in that case a later close() call - // retries the cleanup (the worker may have exited by then). Guarded by - // the synchronized close() method; never read elsewhere. + // True once close() has run its full cleanup sequence INCLUDING a + // CONFIRMED slot-flock release — finishClose() publishes this strictly + // after SlotLock.release() reports success, never before. Pool threads + // treat the flip as "the slot dir is reusable" and free the slot index + // the moment they observe it (QwpWebSocketSender.isSlotLockReleased -> + // SenderPool.reprobeRetiredSlots), so publishing before the release + // would let a replacement engine's SlotLock.acquire collide with the + // still-open fd. Stays false when a close attempt could not confirm + // manager-worker quiescence (or the flock release itself failed) and had + // to leak the ring/watermark/slot lock — in that case a later close() + // call retries the cleanup (the worker may have exited by then). // volatile: latched by finishClose(), but read lock-free by // isCloseCompleted() from pool threads re-probing a retired slot (see the // getter for why it must not synchronize). private volatile boolean closeCompleted; + // Test-only hook run by finishClose() between the terminal cleanup and + // the flock release. Lets a test park the releasing thread inside the + // cleanup/release window and assert that closeCompleted stays false — + // i.e. that completion is never observable while the flock is still + // held. volatile: finishClose may run on the manager worker's exit + // thread while the hook is installed from a test thread. + private volatile Runnable beforeFlockReleaseHook; // Exactly-once claim on the terminal cleanup (finishClose). Contended by // close() and a worker-exit handoff (completeDeferredClose); whoever wins // the CAS runs the cleanup, the loser never touches ring/watermark/flock. @@ -586,9 +600,17 @@ public synchronized void close() { // cleanup runs exactly once and the worker never blocks on the // engine monitor a retried close() holds while joining it. boolean handedOff = false; + boolean registrationFailed = false; try { handedOff = manager.deferUntilWorkerExit(() -> completeDeferredClose(fullyDrained)); } catch (Throwable ignored) { + // Allocation failure (OOM building the cleanup lambda or + // growing the manager's exitCleanups list). Unlike the exact + // false return below, a throw carries NO worker-liveness + // information — the two must never be conflated, or the + // inline cleanup below would release worker-reachable + // resources under a possibly-live worker. + registrationFailed = true; } if (handedOff) { LOG.error("SF manager worker did not quiesce during engine close; ring, watermark " @@ -598,6 +620,23 @@ public synchronized void close() { + "the stale worker on slot {}", sfDir == null ? "" : sfDir); return; } + if (registrationFailed) { + // The handoff never registered and the worker was never + // observed — it must be presumed live and mid service pass. + // Retain every worker-reachable resource (ring, watermark, + // segment files, slot flock) and leave terminalCleanupClaimed + // unclaimed and closeCompleted false, exactly like the + // shared-manager leak branch: manager.close() above already + // stopped the worker loop, so a retried close() converges via + // isWorkerReaped() once the in-flight pass ends. The kernel + // releases the slot flock on process exit regardless. + LOG.error("SF worker-exit handoff registration failed during engine close; " + + "leaking the ring, watermark and slot lock so a possibly-live " + + "worker cannot corrupt a future engine on slot {}. close() may be " + + "invoked again to retry cleanup once the worker has exited.", + sfDir == null ? "" : sfDir); + return; + } // Handoff rejected: the worker loop exited between the failed // bounded join and the registration attempt (both sides share the // manager's lock, so this observation is exact). A worker past @@ -629,8 +668,13 @@ public synchronized void close() { /** * Terminal cleanup: closes the ring and watermark, unlinks drained - * segment files, releases the slot flock, and latches - * {@link #closeCompleted}. The caller must hold the engine monitor and + * segment files, releases the slot flock, and — only once the release + * is confirmed — latches {@link #closeCompleted}. Publish order + * is load-bearing: pools free the slot index the instant they observe + * {@code closeCompleted} (via {@code isSlotLockReleased()}), so it must + * never be visible while the flock fd is still open, or a replacement + * engine races the release and fails acquisition on a live slot. + * The caller must hold the engine monitor and * must have established that the manager worker can no longer touch the * slot directory (reaped, provably exited, or running this on its own * exit path) AND have won the {@link #terminalCleanupClaimed} CAS — the @@ -669,21 +713,64 @@ private void finishClose(boolean fullyDrained) { } catch (Throwable ignored) { } } - closeCompleted = true; } finally { // Reaching finishClose at all requires established quiescence, so // releasing the flock is safe even if a step above threw. Leaking // it would strand the slot until process exit for no reason. + // + // ORDER MATTERS: release the flock FIRST, verify it, and only + // then publish closeCompleted. Pools read isCloseCompleted() as + // "the slot dir is reusable" and free the slot index the moment + // it flips; publishing before Files.close(fd) completes would + // open a window where a replacement engine's SlotLock.acquire + // collides with the still-open fd and fails with a spurious + // "slot already in use". + Runnable hook = beforeFlockReleaseHook; + if (hook != null) { + try { + hook.run(); + } catch (Throwable ignored) { + // test-only; must never block the release + } + } + boolean released; if (slotLock != null) { try { - slotLock.close(); + released = slotLock.release(); } catch (Throwable ignored) { - // best-effort; flock is also released by kernel on process exit + released = false; } + } else { + released = true; + } + if (released) { + closeCompleted = true; + } else { + // Unconfirmed release: the flock may still be held, so keep + // closeCompleted false — the owning pool keeps the slot + // retired (leakedSlots accounting) instead of reusing a + // possibly-locked dir. The kernel releases the flock on + // process exit regardless. + LOG.error("SF slot flock release failed during engine close; keeping " + + "closeCompleted=false so the pool cannot reuse a possibly " + + "still-locked slot dir; the kernel releases the flock on " + + "process exit [slot={}]", sfDir == null ? "" : sfDir); } } } + /** + * Installs a hook that {@link #finishClose} runs between the terminal + * cleanup and the slot-flock release. Test-only: makes the otherwise + * microsecond-wide cleanup/release window deterministic so tests can + * assert {@link #isCloseCompleted()} stays false until the release is + * confirmed. + */ + @TestOnly + public void setBeforeFlockReleaseHook(Runnable hook) { + this.beforeFlockReleaseHook = hook; + } + /** * Runs on the manager worker's exit path when {@link #close()} handed * cleanup ownership to the worker (see {@code deferUntilWorkerExit}). @@ -705,10 +792,13 @@ private void completeDeferredClose(boolean fullyDrained) { } /** - * Whether {@link #close()} completed all cleanup, including releasing the - * SF slot lock. A false value after close means manager-worker quiescence - * could not be confirmed and the worker-reachable resources were retained - * deliberately — either handed to the worker's exit path (owned manager), + * Whether {@link #close()} completed all cleanup, including a + * confirmed release of the SF slot lock — the flip is published + * strictly after the flock fd is closed, so observing {@code true} + * guarantees the slot dir is acquirable by a replacement engine. A false + * value after close means manager-worker quiescence could not be + * confirmed (or the flock release itself failed) and the + * worker-reachable resources were retained deliberately — either handed to the worker's exit path (owned manager), * which flips this to true the moment the worker's in-flight pass * finishes, or leaked until a retried close() (shared manager). Owners * must not reuse the slot while this is false; pools may re-probe it to diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 0b8379de..1b72d157 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -116,15 +116,43 @@ public String slotDir() { return slotDir; } - @Override - public void close() { - // Closing the fd releases the lock. We do NOT remove the .lock - // file or the .lock.pid sidecar — a stale PID is harmless (next - // acquirer overwrites .lock.pid on success). - if (fd >= 0) { - Files.close(fd); + /** + * Releases the flock by closing the lock fd and reports whether the + * release was confirmed. Closing the fd releases the lock. We do + * NOT remove the {@code .lock} file or the {@code .lock.pid} sidecar — + * a stale PID is harmless (next acquirer overwrites {@code .lock.pid} + * on success). + *

    + * On close failure the fd is retained: the flock may still be + * held by this process, so forgetting the fd would misreport the lock + * state and forfeit any chance of a later retry. Idempotent — once + * released, subsequent calls return {@code true}. + *

    + * Owners that gate a "slot dir is reusable" signal on the release + * (e.g. {@code CursorSendEngine.finishClose} publishing + * {@code closeCompleted}) must call this and check the result rather + * than {@link #close()}, which is best-effort by contract. + * + * @return {@code true} if the fd was closed (or was already released), + * {@code false} if the OS reported a close failure and the + * flock may still be held + */ + public boolean release() { + if (fd < 0) { + return true; + } + if (Files.close(fd) == 0) { fd = -1; + return true; } + return false; + } + + @Override + public void close() { + // QuietCloseable contract: best-effort, no signal. Callers that + // must confirm the release use release() and check the result. + release(); } private static String readHolder(String pidPath) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java new file mode 100644 index 00000000..cd7fdff0 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -0,0 +1,176 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Regression test for the completion/release ordering in + * {@code CursorSendEngine.finishClose()}: {@code closeCompleted} must be + * published strictly AFTER the slot flock release is confirmed, never + * before. + * + *

    The bug being pinned: {@code closeCompleted = true} used to be written + * before {@code slotLock} was closed. A pool thread could observe completion + * through {@code QwpWebSocketSender.isSlotLockReleased()}, free the slot + * index in {@code SenderPool.reprobeRetiredSlots()}, and admit a replacement + * sender whose {@code SlotLock.acquire} then collided with the still-open + * flock fd — a spurious "sf slot already in use" construction failure naming + * the process's own pid as the holder. + * + *

    The window between the publish and the {@code Files.close(fd)} is + * microseconds wide in the wild; {@code setBeforeFlockReleaseHook} makes it + * deterministic. The closing thread is parked between terminal cleanup and + * the flock release, and the test asserts from outside the window's two + * halves of the contract: + *

      + *
    • inside the window: {@code isCloseCompleted()} is still false AND a + * fresh {@code SlotLock.acquire} on the slot fails (proving the flock + * is genuinely held — i.e. reporting completion here would have been + * a lie);
      • + *
    • after the window: {@code isCloseCompleted()} is true AND a fresh + * {@code SlotLock.acquire} succeeds (completion still implies + * reusability — the reorder did not break the happy path).
      • + *
    + */ +public class EngineClosePublishAfterFlockReleaseTest { + + private String sfDir; + + @Before + public void setUp() { + sfDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-close-publish-order-" + System.nanoTime()).toString(); + } + + @After + public void tearDown() { + if (sfDir == null) return; + long find = Files.findFirst(sfDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(sfDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(sfDir); + } + + @Test(timeout = 30_000L) + public void testCloseCompletedPublishedOnlyAfterConfirmedFlockRelease() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + CountDownLatch inWindow = new CountDownLatch(1); + CountDownLatch proceed = new CountDownLatch(1); + engine.setBeforeFlockReleaseHook(() -> { + inWindow.countDown(); + try { + // Bounded so a failed assertion on the main thread can + // never wedge the closer past the test timeout. + proceed.await(20, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + AtomicReference closerError = new AtomicReference<>(); + Thread closer = new Thread(() -> { + try { + engine.close(); + } catch (Throwable t) { + closerError.set(t); + } + }, "engine-closer"); + closer.start(); + + try { + assertTrue("closer thread never reached the cleanup/release window", + inWindow.await(10, TimeUnit.SECONDS)); + + // Inside the window: terminal cleanup has run, the flock has + // NOT been released. Completion must not be observable yet — + // this is the exact read a pool thread performs before + // freeing the slot index. + assertFalse("closeCompleted was published before the flock release; " + + "a pool observing this would free the slot index and a " + + "replacement sender would collide with the still-held flock", + engine.isCloseCompleted()); + + // Prove the window is real: the flock is genuinely still + // held, so acquisition by a "replacement engine" fails. + try { + SlotLock probe = SlotLock.acquire(sfDir); + probe.close(); + fail("scaffolding error: expected the slot flock to still be " + + "held inside the pre-release window, but a fresh " + + "SlotLock.acquire succeeded"); + } catch (IllegalStateException expected) { + // good — slot is locked, which is why completion must + // not have been published yet. + } + } finally { + proceed.countDown(); + closer.join(10_000L); + } + assertFalse("closer thread did not finish", closer.isAlive()); + assertNull("engine.close() threw", closerError.get()); + + // After the window: the release is confirmed, so completion must + // now be latched, and completion must still imply reusability. + assertTrue("closeCompleted must latch once the flock release is confirmed", + engine.isCloseCompleted()); + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — completion implies the slot dir is acquirable. + } catch (IllegalStateException stillHeld) { + fail("closeCompleted reported true but the slot flock is still held: " + + stillHeld.getMessage()); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 644a7463..0d4b9f43 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -103,6 +103,25 @@ public void testCloseReleasesLock() throws Exception { }); } + @Test + public void testReleaseConfirmsAndIsIdempotent() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/verified-release"; + SlotLock lock = SlotLock.acquire(slot); + assertTrue("first release must confirm success", lock.release()); + // Idempotent: an already-released lock keeps reporting true — + // callers gating a "slot reusable" signal on it must never see + // a spurious false after a confirmed release. + assertTrue("repeat release must stay true", lock.release()); + // close() after release() is a safe no-op (QuietCloseable path). + lock.close(); + // Confirmed release means the slot is genuinely acquirable. + try (SlotLock again = SlotLock.acquire(slot)) { + assertEquals(slot, again.slotDir()); + } + }); + } + @Test public void testTwoDifferentSlotsCoexist() throws Exception { TestUtils.assertMemoryLeak(() -> { From 7b9924c98d1ef3d1722b95833fd0160e8282cd1d Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 21:40:41 +0100 Subject: [PATCH 09/26] fix(qwp): retain startup-retired recoverers so a late flock release restores pool capacity isSlotLockReleased() is no longer a one-shot snapshot: deferred engine cleanup on a worker/I/O-thread exit path can release the SF slot flock after close() returned. The runtime reclaim paths (discardBroken/reapIdle via reclaimSlot) already keep such slots in retiredSlots and re-probe them, but the in-range startup-recovery pass only ticked leakedSlots and dropped the recoverer, making the retirement permanent even after the release -- fatal at maxSize=1, where every later borrow timed out until process restart. Hand the retained recoverer out of drainCandidateSlotForRecovery (retainedOut replaces the flockHeld boolean) and add it to retiredSlots alongside the leakedSlots tick, so the existing reprobeRetiredSlots() drivers (capacity-starved borrow, housekeeper tick) recover the capacity once the worker finally exits. Out-of-range recoverers stay excluded: they carry no leakedSlots tick and freeSlotIndex(idx) would index past the slotInUse array. --- .../io/questdb/client/impl/SenderPool.java | 62 ++++++---- .../client/test/impl/SenderPoolSfTest.java | 106 +++++++++++++++++- 2 files changed, 142 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 798a931a..4d56a6af 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -179,12 +179,15 @@ public final class SenderPool implements AutoCloseable { // retiredSlots and returns any index whose flock has since dropped. // Guarded by lock; only ever ticks for SF slots. private int leakedSlots; - // The retired slots behind the leakedSlots count (runtime reclaim paths - // only; startup-recovery retirements stay permanent — their transient - // recoverer sender is out of scope by retire time). Re-probed by - // reprobeRetiredSlots() so a late flock release (deferred engine cleanup - // on a worker exit path) restores the pool's capacity instead of - // ratcheting it down until process exit. Guarded by lock. + // The retired slots behind the leakedSlots count: runtime reclaim paths + // (discardBroken/reapIdle via reclaimSlot) and the in-range startup- + // recovery pass (recoverOneSlotStep, which retains the recoverer slot for + // exactly this purpose). Re-probed by reprobeRetiredSlots() so a late + // flock release (deferred engine cleanup on a worker exit path) restores + // the pool's capacity instead of ratcheting it down until process exit. + // Out-of-range startup recoverers are NEVER added: they carry no + // leakedSlots tick and their index has no slotInUse entry to free. + // Guarded by lock. private final ArrayList retiredSlots = new ArrayList<>(); // SF slots currently held by the in-range startup-recovery pass // (recoverOneSlotStep): each is reserved under `lock` for the @@ -480,7 +483,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryComplete = true; return false; } - final boolean[] flockHeld = new boolean[1]; + final SenderSlot[] retained = new SenderSlot[1]; // Pass 1: in-range managed slots [0, maxSize). Skip live and empty slots // cheaply; spend the step on the first slot that actually holds data. @@ -526,23 +529,29 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { // A real candidate -> spend the step on it. Advance the cursor first // so a resume never reprocesses this index. recoveryInRangeNext++; - boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, flockHeld); + boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, retained); lock.lock(); try { // Release the recovery reservation accounting; from here either // leakedSlots (retire) or the freed index carries the cap math. recoveringSlots--; - if (flockHeld[0]) { + if (retained[0] != null) { // close() retained the flock because an I/O or manager - // worker did not stop. Retire the slot permanently (mirror + // worker did not stop. Retire the slot (mirror // discardBroken/reapIdle): keep slotInUse[i] set and count it // in leakedSlots so the borrow() cap math accounts for the // lost capacity and no later borrow ever reuses the - // still-locked dir. + // still-locked dir. Keep the recoverer in retiredSlots so + // reprobeRetiredSlots() restores the capacity once the + // deferred engine cleanup releases the flock — without it + // the retirement would be permanent even after the release + // (fatal at maxSize=1: every later borrow would time out). leakedSlots++; - LOG.warn("startup SF recovery: slot {} retired permanently: delegate close() returned with " + retiredSlots.add(retained[0]); + LOG.warn("startup SF recovery: slot {} retired: delegate close() returned with " + "the flock still held (I/O or manager worker did not stop); pool capacity reduced by 1, " - + "now {} of {} usable [leakedSlots={}]", + + "now {} of {} usable [leakedSlots={}]; the slot is re-probed and recovered " + + "if the worker releases the flock later", i, maxSize - leakedSlots, maxSize, leakedSlots); } else { slotInUse[i] = false; @@ -586,15 +595,17 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { if (!OrphanScanner.isCandidateOrphan(slotPath)) { continue; } - boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, flockHeld); - if (flockHeld[0]) { + boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, retained); + if (retained[0] != null) { // Out of the pool's [0, maxSize) capacity range: there is no // slotInUse entry to retire and no future borrow targets this // dir, so a still-held flock only leaks this recoverer's // worker-reachable resources (a best-effort teardown loss, // logged). Crucially we do // NOT touch leakedSlots -- that would wrongly shrink the - // in-range pool capacity. + // in-range pool capacity -- and we do NOT add to retiredSlots: + // there is no capacity to recover, and freeSlotIndex(idx) + // would index past the slotInUse array (sized maxSize). LOG.warn("startup SF recovery: out-of-range slot {} closed with the flock still held " + "(I/O or manager worker did not stop); its data is durable on disk for a later attempt", slotPath); @@ -616,17 +627,20 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { * (whose {@link #defaultSender} derives the dir {@code -slotIndex}), * drains its unacked data, and closes the delegate. Shared by both recovery * passes -- the in-range pass and the out-of-range pass -- which differ only - * in their slot bookkeeping, handled by the caller via {@code flockHeld}. + * in their slot bookkeeping, handled by the caller via {@code retainedOut}. * - * @param flockHeld single-element out-param set to {@code true} iff a - * recoverer was built and its {@code close()} returned with - * the flock still held because a worker did not stop + * @param retainedOut single-element out-param set to the recoverer iff one + * was built and its {@code close()} returned with the + * flock still held because a worker did not stop; the + * in-range caller keeps it in {@link #retiredSlots} so a + * late flock release can be re-probed. {@code null} when + * the flock was released (or no recoverer was built). * @return {@code true} if a build/drain failure occurred that will very * likely repeat for every remaining slot, so the caller should stop scanning */ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, - long remainingMillis, boolean[] flockHeld) { - flockHeld[0] = false; + long remainingMillis, SenderSlot[] retainedOut) { + retainedOut[0] = null; // Hoisted so the flock check after the try can consult it: // createRecoverer() takes the slot flock on -slotIndex, and // delegate().close() can retain it when an I/O or manager worker does @@ -678,8 +692,8 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, LOG.warn("startup SF recovery: scan failed for slot {} ({})", slotPath, scanErr.toString()); } - if (recoverer != null) { - flockHeld[0] = !flockReleased(recoverer); + if (recoverer != null && !flockReleased(recoverer)) { + retainedOut[0] = recoverer; } return stopScan; } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 75aca494..60519571 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -875,7 +875,7 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th // C1 regression: the startup recovery loop MUST mirror discardBroken / // reapIdle. When a recoverer's delegate close() returns with the SF // flock still held (the I/O thread refused to stop), the recovered slot - // index must be retired permanently (leakedSlots++, slotInUse stays + // index must be retired (leakedSlots++, slotInUse stays // set) -- NOT freed. Freeing it would let a later borrow re-pick the // still-locked dir and resurrect "sf slot already in use", the exact // failure class this PR exists to kill. Pre-fix the recovery finally @@ -951,7 +951,7 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th try { Assert.assertTrue("borrow must use a fresh slot dir", Files.exists(slot("default-1"))); - // Capacity is permanently reduced by the leaked slot: + // Capacity stays reduced while the flock is held: // max=2, one leaked + one live => the next borrow times // out rather than colliding on the locked dir. try { @@ -979,6 +979,108 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th }); } + @Test + public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Exception { + // Recovery twin of + // testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld: a + // slot retired by STARTUP recovery (recoverer close() returned with + // the flock still held) must not stay lost until process exit. + // isSlotLockReleased() is no longer a one-shot snapshot -- deferred + // engine cleanup on a worker exit path can release the flock later -- + // so the pool must keep the recoverer in retiredSlots and re-probe it. + // Pre-fix, startup recovery only ticked leakedSlots and dropped the + // recoverer: at maxSize=1 every later borrow timed out forever even + // after the flock dropped. This test is RED until the recoverer is + // retained. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0 so startup recovery + // treats it as a candidate orphan and builds a recoverer. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: maxSize=1 -- the worst case, where the single slot's + // retirement starves the whole pool. Forge the retention exactly + // like the retire test (closed=true makes the recovery drain and + // close a no-op, so the real flock stays held and slotLockReleased + // stays false), but only for the FIRST build of slot 0: the + // post-recovery borrow below must get a working delegate on the + // recovered index. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + setBooleanField(real, "closed", true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 500, factory)) { + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + // While the flock is genuinely held, borrows time out: the + // cap-check re-probe finds the flock still reported held. + try { + pool.borrow(); + Assert.fail("borrow must time out while the slot is retired"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out")); + } + + // The late release: the "wedged worker" finishes and the + // flock genuinely drops (un-forge and close the recoverer + // for real; in production this flip comes from + // isSlotLockReleased() re-probing the retained engine after + // the worker exited). + Sender recoverer = forged.get(); + setBooleanField(recoverer, "closed", false); + recoverer.close(); + + // The capacity-starved borrow must re-probe the startup- + // retired slot, recover its capacity, and create on the + // freed index -- no housekeeper tick required. + PooledSender b = pool.borrow(); + try { + Assert.assertEquals("borrow must recover the startup-retired slot's capacity", + 0, pool.leakedSlotCount()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + Assert.assertTrue("recovered index 0 must carry the new borrow", slotInUse[0]); + Assert.assertEquals(1, countSlotDirs()); + } finally { + b.close(); + } + } + } + }); + } + // ---------------------------------------------------------------------- // Concurrency stress: borrow/return churn must never collide on a slot. // ---------------------------------------------------------------------- From d990470aaeb239bfa83b362c9be1661aed41122d Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 23:55:08 +0100 Subject: [PATCH 10/26] fix(qwp): give capacity-starved borrows a final retired-slot probe before the timeout check borrow() ran the terminal timeout check before reprobeRetiredSlots(), so a zero-timeout (try-once) borrow threw without its one probe, and a borrower whose awaitNanos budget expired mid-wait timed out on capacity that a deferred engine cleanup had already returned (the delegate-side flock release never signals slotReleased). Hoist the probe above the timeout check so both paths recover the capacity instead of failing. Also pre-size retiredSlots to maxSize: every entry keeps a distinct in-range slot index reserved, so add() can never grow the backing array -- a retire (leakedSlots++ then add, under lock) can no longer fail on allocation and strand a counted-but-untracked slot that reprobeRetiredSlots() could never recover. Tests: - testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing (red pre-fix) - testParkedBorrowerGetsFinalProbeAfterBudgetExpiry (red pre-fix) - testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge: full-stack retire/recover cycle with no forged flags -- real wedged manager worker, real timed-out close handoff, real flock release, and a re-borrow on the recovered index proving the slot dir is genuinely reusable --- .../io/questdb/client/impl/SenderPool.java | 29 +- .../client/test/impl/SenderPoolSfTest.java | 308 ++++++++++++++++++ 2 files changed, 329 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 4d56a6af..34b59191 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -187,8 +187,12 @@ public final class SenderPool implements AutoCloseable { // the pool's capacity instead of ratcheting it down until process exit. // Out-of-range startup recoverers are NEVER added: they carry no // leakedSlots tick and their index has no slotInUse entry to free. - // Guarded by lock. - private final ArrayList retiredSlots = new ArrayList<>(); + // Pre-sized to maxSize (every entry keeps a distinct in-range slot index + // reserved, so size can never exceed maxSize): add() never grows the + // backing array, so a retire (leakedSlots++ then add, under lock) cannot + // fail on allocation and strand a counted-but-untracked slot that + // reprobeRetiredSlots() could never recover. Guarded by lock. + private final ArrayList retiredSlots; // SF slots currently held by the in-range startup-recovery pass // (recoverOneSlotStep): each is reserved under `lock` for the // duration of its drain and counted in the borrow() cap check so a @@ -306,6 +310,7 @@ public SenderPool( this.maxLifetimeMillis = maxLifetimeMillis; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); + this.retiredSlots = new ArrayList<>(maxSize); this.slotReleased = lock.newCondition(); // Probe the config once, up front: this validates it eagerly (so a // bad config fails at construction even when minSize == 0) and tells @@ -793,16 +798,24 @@ public PooledSender borrow() { created.bumpGeneration(); return new PooledSender(created, created.generation()); } + // Capacity-starved: re-probe retired slots BEFORE the terminal + // timeout check — a deferred engine cleanup may have released a + // flock since the retire, and the freed index can admit a + // creation right now. The release itself never signals + // slotReleased (it happens in the delegate on a worker/I/O-thread + // exit path, volatile writes only), so this poll is the only way + // a borrower learns of it. Ordering matters twice over: a + // zero-timeout (try-once) borrow must get its one probe before + // throwing, and a borrower whose awaitNanos budget just expired + // must get a final probe on its wake-up pass instead of timing + // out on capacity that has already come back. + if (reprobeRetiredSlots()) { + continue; + } if (remainingNanos <= 0) { throw new LineSenderException( "timed out waiting for a Sender from the pool after " + acquireTimeoutMillis + "ms"); } - // Capacity-starved: before parking, re-probe retired slots — a - // deferred engine cleanup may have released a flock since the - // retire, and the freed index can admit a creation right now. - if (reprobeRetiredSlots()) { - continue; - } try { remainingNanos = slotReleased.awaitNanos(remainingNanos); } catch (InterruptedException e) { diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 60519571..abe95d56 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -30,7 +30,9 @@ import ch.qos.logback.core.read.ListAppender; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import io.questdb.client.std.Files; @@ -53,6 +55,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -771,6 +774,116 @@ public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { }); } + @Test + public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws Exception { + // Full-stack twin of the two forged-flag recovery tests above: no + // reflection-forged slotLockReleased anywhere. The REAL mechanism is + // driven end to end — the delegate's owned SegmentManager worker is + // wedged mid service pass (test hook), the delegate's close() takes + // the real timed-out-join → worker-exit handoff path, the pool + // retires the slot off the delegate's genuine isSlotLockReleased() + // report, the worker's deferred cleanup releases the real flock, and + // the housekeeper re-probe restores capacity — proving the recovered + // index is genuinely reusable by borrowing on it again (a forged flag + // would pass the accounting asserts but collide on the flock here). + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + Sender delegate = getDelegate(a); + CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); + Assert.assertNotNull("SF delegate must own a cursor engine", engine); + SegmentManager manager = (SegmentManager) getField(engine, "manager"); + + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + try { + // Park the manager worker inside a service pass for + // the delegate's ring. The trim-sync point is reached + // on every ~1ms tick, so this wedges deterministically. + manager.setBeforeTrimSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, new AssertionError( + "timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + Assert.assertTrue("manager worker never entered a service pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Real teardown against the wedged worker: the + // delegate's engine close times out its bounded join, + // hands cleanup to the worker's exit path, and reports + // the retained flock; the pool must retire the slot. + manager.setWorkerJoinTimeoutMillis(50L); + invokeDiscardBroken(pool, a); + Assert.assertEquals( + "pool must retire the slot while the delegate's manager " + + "worker holds the deferred cleanup", + 1, pool.leakedSlotCount()); + Assert.assertFalse("engine cleanup must still be pending", + engine.isCloseCompleted()); + + // Un-wedge: the worker finishes its pass, exits (its + // loop was stopped by the close), and runs the + // deferred cleanup that releases the real flock. + releaseWorker.countDown(); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError( + "deferred engine cleanup never ran on manager-worker exit"); + } + Thread.sleep(1); + } + + // The housekeeper tick is the recovery driver. + pool.reapIdle(); + Assert.assertEquals("recovered slot must leave the leaked count", + 0, pool.leakedSlotCount()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + Assert.assertFalse("recovered slot index 0 must return to the free set", + slotInUse[0]); + + // The proof a forged flag cannot fake: both indices — + // including the recovered one, whose flock was really + // dropped by the worker-exit cleanup — admit live + // senders again. + PooledSender b = pool.borrow(); + PooledSender c = pool.borrow(); + try { + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + b.close(); + } + if (hookErr.get() != null) { + throw new AssertionError("trim hook failed", hookErr.get()); + } + } finally { + manager.setBeforeTrimSyncHook(null); + releaseWorker.countDown(); + } + } + } + }); + } + // ---------------------------------------------------------------------- // Recovery: stable slot ids let a re-created pool re-adopt unacked data. // ---------------------------------------------------------------------- @@ -1081,6 +1194,201 @@ public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Except }); } + @Test + public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Exception { + // Boundary twin of testStartupRetiredSlotRecoveredAfterLateFlockRelease: + // acquireTimeoutMillis=0 is a valid try-once borrow (builder rejects only + // < 0). Pre-fix, borrow() ran the terminal timeout check BEFORE + // reprobeRetiredSlots(), so a zero-budget borrow threw "timed out" + // without its one probe -- even when the retired slot's flock had + // already dropped and a probe would have restored capacity and admitted + // a creation. Deterministic: no housekeeper runs in this test, so + // borrow() is the only reprobe driver. Recovery is driven manually via + // the deferred-pool step helper because the inline path reuses + // acquireTimeoutMillis as its recovery budget -- 0 would skip recovery + // outright. This test is RED until the probe is hoisted above the + // timeout check. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0 so startup recovery + // treats it as a candidate orphan and builds a recoverer. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: maxSize=1, zero acquire budget, deferred recovery driven + // step-by-step (the housekeeper's per-tick unit, budgeted by + // RECOVERY_DRAIN_BUDGET_MILLIS, not the zero acquire budget). Forge + // the retirement (closed=true makes the recovery drain and close a + // no-op, so the real flock stays held), then release the flock for + // real BEFORE borrowing: the recovery is discoverable, but only via + // a probe. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + setBooleanField(real, "closed", true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = newDeferredPoolWithFactory(cfg2, 0, 1, 0, factory)) { + //noinspection StatementWithEmptyBody + while (invokeRunStartupRecoveryStep(pool)) { + // drive the whole backlog, one housekeeper-tick unit at a time + } + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + // The late release: un-forge and close the recoverer for + // real. The flock genuinely drops, but nothing signals the + // pool -- the release happens in the delegate, volatile + // writes only. + Sender recoverer = forged.get(); + setBooleanField(recoverer, "closed", false); + recoverer.close(); + + // Try-once borrow: its single pass must probe, recover the + // capacity, and create on the freed index. Pre-fix this + // threw "timed out" without ever probing. + PooledSender b = pool.borrow(); + try { + Assert.assertEquals("zero-timeout borrow must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + } finally { + b.close(); + } + } + } + }); + } + + @Test + public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception { + // Release-during-wait twin of the zero-timeout test. A borrower parks + // in awaitNanos while the retired slot's flock is genuinely held; the + // deferred cleanup then releases the flock mid-wait. Nothing signals + // slotReleased (the release is delegate-side, volatile writes only), so + // the borrower sleeps out its full budget. Pre-fix its wake-up pass hit + // the terminal timeout check before reprobeRetiredSlots() and threw -- + // missing capacity that had already come back. Post-fix the wake-up + // pass probes first, recovers the index, and the creation is admitted. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: maxSize=1, generous budget so the mid-wait release lands + // comfortably inside it. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + setBooleanField(real, "closed", true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 2_000, factory)) { + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + // Borrower parks: its pre-park probe correctly fails while + // the flock is genuinely held. + AtomicReference failure = new AtomicReference<>(); + AtomicReference borrowed = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + borrowed.set(pool.borrow()); + } catch (Throwable e) { + failure.set(e); + } + }); + borrower.start(); + + // Let the borrower enter borrow() and park. If a CI stall + // delays it past the release below, the pre-park probe + // recovers instead and the test degrades to a pass -- never + // a flaky failure. + Thread.sleep(300); + + // Mid-wait release: flock drops, no signal reaches the pool. + Sender recoverer = forged.get(); + setBooleanField(recoverer, "closed", false); + recoverer.close(); + + borrower.join(10_000); + Assert.assertFalse("borrower must have finished", borrower.isAlive()); + if (failure.get() != null) { + throw new AssertionError( + "borrower must recover the retired slot on its wake-up pass, not time out", + failure.get()); + } + PooledSender b = borrowed.get(); + Assert.assertNotNull("borrower must have obtained a sender", b); + try { + Assert.assertEquals("wake-up probe must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + } finally { + b.close(); + } + } + } + }); + } + // ---------------------------------------------------------------------- // Concurrency stress: borrow/return churn must never collide on a slot. // ---------------------------------------------------------------------- From 7df62d5d6925eedee0a6461c5e18bd73b70d0450 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Fri, 10 Jul 2026 23:55:27 +0100 Subject: [PATCH 11/26] test(qwp): close the remaining engine/manager shutdown coverage gaps Deterministic regression tests for the shutdown paths the coverage review flagged as untested: - SegmentManagerCloseRaceTest#testStaleSnapshotEntrySkippedAfterDeregisterBeforeServiceClaim: a snapshot entry deregistered before the worker claims it must be skipped at claim time (serviced rings recorded from the worker's own trim-sync point via inService, so the assertion is exact -- no sleeps). - SegmentManagerCloseRaceTest#testWorkerAloneFreesPathScratchAfterTimedOutClose: after a timed-out close hands pathScratch to the worker, the worker's exit block alone must free it -- no retried close() runs in production, so the sibling test's retry-then-assert shape masked a regression here. - CursorSendEngineSlotReacquisitionTest#testTerminalCleanupRunsExactlyOnceWhenRetriedCloseRacesWorkerHandoff: a retried close() racing the worker parked mid-finishClose must lose the terminalCleanupClaimed CAS -- no double cleanup, no premature completion, flock untouched until the worker's release. - CursorSendEngineSlotReacquisitionTest#testMemoryModeOwnedCloseHandsCleanupToWorkerExit: memory-mode (null sfDir/slotLock/watermark) timed-out close must take the same worker-exit handoff without NPE and free the ring's native memory. - EngineClosePublishAfterFlockReleaseTest#testUnconfirmedFlockReleaseKeepsCloseIncomplete: a failed flock release must never publish closeCompleted, and a retried close() must neither throw nor fabricate completion. - SlotLockTest#testFailedCloseRetainsFdAndReportsFalse: release()==false retains the fd for retry and keeps reporting false while the failure persists. - SlotLockReleasedContractTest#testDelegatedIoThreadEngineCloseFlipsSlotLockReleased: the delegated-I/O-close branch (delegateEngineClose()==true) must retain the engine so isSlotLockReleased() flips true once the I/O thread's exit path releases the flock; the existing forged I/O-refusal test throws from delegateEngineClose() before the retained-engine assignment and can never reach this branch. Mutation-verified: dropping the retainedEngine assignment fails the test with the pinned message. --- .../client/SlotLockReleasedContractTest.java | 181 +++++++++++++ ...CursorSendEngineSlotReacquisitionTest.java | 250 ++++++++++++++++++ ...gineClosePublishAfterFlockReleaseTest.java | 66 +++++ .../cursor/SegmentManagerCloseRaceTest.java | 236 +++++++++++++++++ .../qwp/client/sf/cursor/SlotLockTest.java | 49 ++++ 5 files changed, 782 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index e8ef2670..0a6c0895 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -24,10 +24,13 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.Sender; +import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; @@ -305,6 +308,162 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception }); } + /** + * Delegated-I/O-close recovery path: when {@code close()} bails early + * because the I/O thread refuses to stop, the engine close is delegated + * to that thread's exit path ({@code delegateEngineClose()} returned + * true) and the sender must retain the engine for re-probing — + * {@code isSlotLockReleased()} stays false while the thread lives, then + * flips true the moment the thread's exit path completes the engine + * close and releases the flock. Pre-fix, this branch discarded the + * engine reference, so the late release was permanently invisible to a + * pool that had retired the slot. The existing forged I/O-refusal test + * can never reach this branch: its sabotaged loop throws from + * {@code delegateEngineClose()} itself (nulled latch), before the + * retained-engine assignment. + *

    + * The wedge is the same deterministic one CursorWebSocketSendLoop's C5 + * test uses: the I/O thread sits in a blocking "connect" (a + * ReconnectFactory immune to unpark, never interrupted by loop.close()), + * and the closer thread's pre-set interrupt flag makes + * {@code shutdownLatch.await()} throw immediately — the real production + * path to {@code ioThreadStopped = false}. + */ + @Test(timeout = 30_000L) + public void testDelegatedIoThreadEngineCloseFlipsSlotLockReleased() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-delegated-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + final CountDownLatch enteredConnect = new CountDownLatch(1); + final CountDownLatch releaseConnect = new CountDownLatch(1); + final AtomicReference ioThreadRef = new AtomicReference<>(); + final StubWebSocketClient stubClient = new StubWebSocketClient(); + // Healthy owned manager: once the wedged I/O thread is released, + // its exit-path engine.close() completes normally and releases + // the flock — the flip this test pins. + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + CursorWebSocketSendLoop loop = null; + QwpWebSocketSender wss = null; + try { + // Stand-in for a blocking native connect(2): entered by the + // loop's I/O thread, immune to unpark, never interrupted by + // loop.close(). + CursorWebSocketSendLoop.ReconnectFactory stuckConnect = () -> { + ioThreadRef.set(Thread.currentThread()); + enteredConnect.countDown(); + releaseConnect.await(); + return stubClient; + }; + loop = new CursorWebSocketSendLoop( + null /* async-initial-connect: the I/O thread drives the connect */, + engine, 0L, 1_000L, + stuckConnect, + 5_000L, 100L, 5_000L, false); + loop.start(); + Assert.assertTrue("I/O thread never reached the connect factory", + enteredConnect.await(5, TimeUnit.SECONDS)); + + wss = QwpWebSocketSender.createForTesting("localhost", 1); + wss.setCursorEngine(engine, true); + setField(wss, "cursorSendLoop", loop); + + // Drive the real early-bail close() on a thread whose pending + // interrupt lands in loop.close()'s shutdownLatch.await(). + AtomicReference closeFailure = new AtomicReference<>(); + QwpWebSocketSender wssRef = wss; + Thread closer = new Thread(() -> { + Thread.currentThread().interrupt(); + try { + wssRef.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "delegated-close-closer"); + closer.setDaemon(true); + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("closer thread did not finish", closer.isAlive()); + Assert.assertNotNull("close() must surface the failed I/O-thread stop", + closeFailure.get()); + + // The I/O thread is still wedged: the flock is retained and + // must be reported retained. + Assert.assertFalse( + "isSlotLockReleased() must be false while the delegated engine " + + "close is pending on the wedged I/O thread", + wss.isSlotLockReleased()); + Assert.assertFalse("engine close must not have run yet", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot became acquirable while the delegated engine close " + + "was still pending"); + } catch (Exception expected) { + // good — the flock is genuinely held. + } + + // Un-wedge the connect. The I/O thread exits; its exit path + // runs the delegated engine.close(), which releases the flock. + releaseConnect.countDown(); + Thread ioThread = ioThreadRef.get(); + Assert.assertNotNull(ioThread); + ioThread.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("I/O thread did not exit after the connect returned", + ioThread.isAlive()); + + // The recovery contract under test: the getter re-probes the + // retained engine, so the late release MUST become visible — + // this is what lets SenderPool recover the retired slot. + Assert.assertTrue("delegated engine close must have completed on the " + + "I/O thread's exit path", + engine.isCloseCompleted()); + Assert.assertTrue( + "isSlotLockReleased() must flip true once the delegated engine " + + "close released the flock — otherwise the pool retires the " + + "slot's capacity until process exit", + wss.isSlotLockReleased()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the delegated close", probe); + } + } finally { + releaseConnect.countDown(); + // Reap the loop's bookkeeping now that the I/O thread is gone + // (close() threw mid-teardown, so ioThread was left set). + Thread.interrupted(); + if (loop != null) { + try { + loop.close(); + } catch (Throwable ignored) { + } + } + if (engine != null && !engine.isCloseCompleted()) { + try { + engine.close(); + } catch (Throwable ignored) { + } + } + // The early-return close() deliberately leaked the resources + // the (then-running) I/O thread might touch; free the same set + // the post-guard tail would have freed. + if (wss != null) { + freeFieldQuietly(wss, "buffer0"); + freeFieldQuietly(wss, "buffer1"); + freeFieldQuietly(wss, "client"); + freeFieldQuietly(wss, "errorDispatcher"); + freeFieldQuietly(wss, "progressDispatcher"); + freeFieldQuietly(wss, "connectionDispatcher"); + } + stubClient.close(); + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + // ------------------------------------------------------------------ utils private static void freeFieldQuietly(Object target, String name) { @@ -370,6 +529,28 @@ private static void setField(Object target, String name, Object value) throws Ex throw new NoSuchFieldException(name); } + /** + * Minimal concrete {@link WebSocketClient} — never performs I/O. Handed + * to the loop by the stuck-connect factory; the loop's exit path closes + * it (close is idempotent via the superclass). + */ + private static final class StubWebSocketClient extends WebSocketClient { + + StubWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + /** ACKs every binary frame with a running sequence so flush/close drain cleanly. */ private static final class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler { private final AtomicLong nextSeq = new AtomicLong(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index ad25ff78..38cd1bcf 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -43,6 +43,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; /** @@ -523,6 +524,255 @@ public void testOwnedEngineCloseRetainsSlotWhenHandoffRegistrationFails() throws }); } + /** + * Exactly-once contention on the terminal-cleanup claim + * ({@code terminalCleanupClaimed} CAS): after a timed-out owned close + * handed cleanup to the worker's exit path, a retried {@code close()} + * that races the worker MID-{@code finishClose} must neither re-run the + * terminal cleanup (double munmap / double flock release) nor block on + * the worker, nor publish completion on the worker's behalf. The race + * window is made deterministic by parking the worker inside + * {@code finishClose} (via {@code beforeFlockReleaseHook}) while the + * retried close() converges through {@code isWorkerReaped()} and loses + * the CAS. + */ + @Test(timeout = 30_000L) + public void testTerminalCleanupRunsExactlyOnceWhenRetriedCloseRacesWorkerHandoff() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/cas-contention-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + CountDownLatch inFinishClose = new CountDownLatch(1); + CountDownLatch releaseFinishClose = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicInteger finishCloseRuns = new AtomicInteger(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass. + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + // Counts terminal-cleanup executions and parks the FIRST one + // (the worker's deferred cleanup) mid-finishClose, before the + // flock release — the exact window a retried close() races. + engine.setBeforeFlockReleaseHook(() -> { + if (finishCloseRuns.incrementAndGet() == 1) { + inFinishClose.countDown(); + try { + if (!releaseFinishClose.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, new AssertionError( + "timed out waiting for test to release finishClose")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + } + }); + + // Phase 2: rotate onto the spare so the worker parks in the + // next install pass. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: timed-out close — cleanup ownership transfers to + // the worker's exit path. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete while the worker holds the handoff", + engine.isCloseCompleted()); + + // Phase 4: release the pass. The worker exits its loop, wins + // the cleanup CAS, enters finishClose and parks in the hook — + // mid-cleanup, flock still held, completion unpublished. + releaseWorker.countDown(); + Assert.assertTrue("worker never entered the deferred finishClose", + inFinishClose.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, finishCloseRuns.get()); + Assert.assertFalse("completion must not be observable mid-finishClose", + engine.isCloseCompleted()); + + // Phase 5 — the contention under test: a retried close() while + // the worker is parked INSIDE finishClose. The worker loop has + // already exited (workerLoopExited=true precedes the deferred + // cleanups), so the short bounded join reaps the manager state + // and close() converges to the CAS — which it must LOSE, + // returning promptly without touching ring/watermark/flock. + engine.close(); + Assert.assertEquals( + "retried close() re-ran the terminal cleanup while the worker's " + + "deferred cleanup was mid-flight — ring/watermark/flock would " + + "be double-released", + 1, finishCloseRuns.get()); + Assert.assertFalse("retried close() must not publish completion on the worker's behalf", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot lock observable as released while the worker was still " + + "parked before its flock release"); + } catch (Exception expected) { + // good — flock still held by the parked cleanup. + } + + // Phase 6: let the worker finish. Completion publishes, the + // slot becomes acquirable, and the cleanup count stays at 1. + releaseFinishClose.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError("deferred cleanup never completed after release"); + } + Thread.sleep(1); + } + Assert.assertEquals(1, finishCloseRuns.get()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the worker-exit cleanup", probe); + } + // A final close() takes the fast no-op path. + engine.close(); + Assert.assertEquals("post-completion close() must be a no-op", + 1, finishCloseRuns.get()); + if (hookErr.get() != null) { + throw new AssertionError("hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + engine.setBeforeFlockReleaseHook(null); + releaseWorker.countDown(); + releaseFinishClose.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * Memory-mode twin of {@link #testOwnedEngineCloseHandsCleanupToWorkerExit}: + * {@code sfDir == null}, so there is no slot lock, no watermark and no + * segment files — but the ring's malloc'd native segments are still + * worker-reachable, so the timed-out close must take the same handoff + * path with every SF-only resource null. Pins that (a) the handoff + * branch tolerates null slotLock/watermark/sfDir without NPE, (b) the + * close stays incomplete while the worker can still touch the ring, and + * (c) the worker-exit cleanup completes the close and frees the ring's + * native memory (assertMemoryLeak is the leak oracle here). + */ + @Test(timeout = 30_000L) + public void testMemoryModeOwnedCloseHandsCleanupToWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Memory mode: null sfDir, private owned manager — the exact + // shape non-SF async ingest uses. + CursorSendEngine engine = new CursorSendEngine(null, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass. + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: rotate onto the spare; the worker's next tick + // re-enters the (in-memory) install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: timed-out memory-mode close. The worker can still + // touch the ring's native memory, so the close must hand off + // and stay incomplete — releasing the ring here would be a + // use-after-free on the worker's install path. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse( + "memory-mode close must stay incomplete while the worker is mid service pass", + engine.isCloseCompleted()); + + // Phase 4: release the worker; its exit path must run the + // deferred cleanup (null slotLock/watermark/sfDir) and flip + // completion with no further caller action. + releaseWorker.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError( + "deferred memory-mode cleanup never ran on manager-worker exit — " + + "the ring's native segments would leak for the process lifetime"); + } + Thread.sleep(1); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + /** * An engine that owns its manager must use the whole-manager stop/join as * its only quiescence barrier. Calling the per-ring barrier first would diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java index cd7fdff0..5dcf6def 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -32,12 +32,14 @@ import org.junit.Before; import org.junit.Test; +import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -173,4 +175,68 @@ public void testCloseCompletedPublishedOnlyAfterConfirmedFlockRelease() throws E } }); } + + /** + * The error half of the publish-after-release contract: when + * {@code SlotLock.release()} reports failure (the OS refused the fd + * close, so the flock may still be held), {@code closeCompleted} must + * NEVER be published — not by the failing close(), and not by a retried + * close() either (the terminal-cleanup claim is already consumed; the + * design deliberately leaves an unconfirmed release to the kernel's + * process-exit cleanup rather than risking a double release). A pool + * observing {@code isCloseCompleted() == false} keeps the slot retired, + * which is exactly right: the flock genuinely is still held. + *

    + * {@code Files.close} cannot be made to fail through the public API, so + * the lock's fd is swapped to a known-bad descriptor for the close and + * restored (and released for real) afterwards. + */ + @Test(timeout = 30_000L) + public void testUnconfirmedFlockReleaseKeepsCloseIncomplete() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock"); + slotLockField.setAccessible(true); + SlotLock slotLock = (SlotLock) slotLockField.get(engine); + assertNotNull("disk-mode engine must hold a slot lock", slotLock); + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(slotLock); + assertTrue("precondition: live flock fd", realFd >= 0); + try { + // A non-negative fd no process has open: close(2) fails EBADF, + // so finishClose's release() confirmation fails. + fdField.setInt(slotLock, 1_000_000_000); + engine.close(); + assertFalse( + "closeCompleted was published despite an unconfirmed flock release; " + + "a pool observing this would free the slot index while the " + + "flock fd is still open", + engine.isCloseCompleted()); + // The REAL flock is still held — the incomplete report is true. + try { + SlotLock probe = SlotLock.acquire(sfDir); + probe.close(); + fail("slot must not be acquirable while the original flock fd is still open"); + } catch (IllegalStateException expected) { + // good — incomplete close really means "still locked". + } + // A retried close() must neither throw nor re-run the terminal + // cleanup (the claim is consumed) nor suddenly report success. + engine.close(); + assertFalse("retried close() must not fabricate completion after a failed " + + "flock release — the claim is consumed and the kernel owns " + + "the eventual cleanup", + engine.isCloseCompleted()); + } finally { + // Undo the fault and drop the real flock so the test leaves no + // open fd behind (in production the kernel does this at exit). + fdField.setInt(slotLock, realFd); + assertTrue("restored fd must release cleanly", slotLock.release()); + } + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — with the flock genuinely dropped, the slot is reusable. + } + }); + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index 9f6fc225..cee8e41d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -204,6 +204,224 @@ public void testCloseDoesNotFreePathScratchWhenWorkerStillAlive() throws Excepti }); } + /** + * Pins the claim-time registration gate at the top of + * {@code SegmentManager.serviceRing}: a snapshot entry whose ring was + * deregistered BEFORE the worker claims it must be skipped entirely — + * never claimed as {@code inService}, never handed to + * {@code serviceRing0}. This is the exact guarantee + * {@code CursorSendEngine.close()} relies on when it releases the ring, + * watermark and slot flock right after + * {@code awaitRingQuiescence(ring) == true}: the deregistering thread may + * already be freeing those resources, so a stale snapshot entry must not + * be touched at all (spare install, watermark write, drainTrimmable, or + * path building under the slot dir). + *

    + * Deterministic shape: three rings A, B, C registered before start, so + * the worker's first snapshot is exactly [A, B, C]. The worker parks in + * A's spare-install pass; while it is parked, B is deregistered (and a + * quiescence barrier for B must pass immediately — no pass for B is in + * flight). After release the worker walks the rest of its stale + * snapshot: it must skip B and service C. Every serviced ring is + * recorded from inside the worker's own pass (via the trim-sync hook + * reading {@code inService}), so the assertion is exact — no timing + * grace, no sleeps. + */ + @Test(timeout = 15_000L) + public void testStaleSnapshotEntrySkippedAfterDeregisterBeforeServiceClaim() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + SegmentRing[] rings = new SegmentRing[3]; + String[] slots = new String[3]; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Rings the worker actually claimed and serviced, recorded from + // the worker thread itself at the trim-sync point every service + // pass reaches (spare needed or not). + java.util.Set serviced = + java.util.Collections.synchronizedSet( + java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<>())); + boolean managerClosed = false; + try { + for (int i = 0; i < 3; i++) { + slots[i] = tmpDir + "/claim-skip-slot-" + i; + Assert.assertEquals(0, Files.mkdir(slots[i], Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create( + slots[i] + "/sf-initial.sfa", 0L, segSize); + rings[i] = new SegmentRing(initial, segSize); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.setBeforeTrimSyncHook(() -> { + try { + Object ring = readInServiceRing(manager); + if (ring != null) { + serviced.add(ring); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + // Register all three BEFORE start: the worker's first snapshot + // is [A, B, C] and every fresh ring wants a hot spare, so the + // install hook parks the worker inside A's pass. + manager.register(rings[0], slots[0]); + manager.register(rings[1], slots[1]); + manager.register(rings[2], slots[2]); + manager.start(); + Assert.assertTrue("worker did not reach A's install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // B is deregistered while its snapshot entry is still ahead of + // the worker's cursor. The quiescence barrier must pass at + // once: the in-flight pass is A's, not B's — this is the state + // in which an engine owner frees B's resources. + manager.deregister(rings[1]); + Assert.assertTrue("no pass for B is in flight — the barrier must pass immediately", + manager.awaitRingQuiescence(rings[1])); + + releaseWorker.countDown(); + + // Positive marker that the worker walked PAST B's snapshot + // slot: C sits after B in the same snapshot, so once C has + // been serviced, B's claim-or-skip decision has been made. + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!serviced.contains(rings[2])) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("worker never serviced ring C after release"); + } + Thread.sleep(1); + } + + Assert.assertTrue("ring A must have been serviced", serviced.contains(rings[0])); + Assert.assertFalse( + "worker claimed and serviced a snapshot entry that was deregistered " + + "before its pass started — the deregistering thread may already " + + "be releasing the ring/watermark/slot lock, so a stale snapshot " + + "entry must be skipped at claim time", + serviced.contains(rings[1])); + // Defense in depth: nothing may have been installed into the + // deregistered ring either. + Assert.assertNull("no hot spare may be installed into a deregistered ring", + readHotSpare(rings[1])); + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("worker-side hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + manager.setBeforeTrimSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + manager.close(); + } + for (SegmentRing ring : rings) { + if (ring != null) { + ring.close(); + } + } + } + }); + } + + /** + * Pins the scratch-handoff half of the timed-out-close contract in + * isolation: after {@code close()} gives up on the bounded join and hands + * {@code pathScratch} ownership to the worker, the WORKER's exit block + * alone must free the native buffer — with no retried {@code close()} + * ever running. The sibling test + * ({@link #testCloseDoesNotFreePathScratchWhenWorkerStillAlive}) retries + * {@code close()} before asserting the free, so a regression that dropped + * the worker-side free (leaving reclaim to a retry nobody is required to + * make) would stay green there: production owners do NOT retry — a + * timed-out close returns to the pool and the worker is the only thread + * left that can reclaim the allocation. + */ + @Test(timeout = 15_000L) + public void testWorkerAloneFreesPathScratchAfterTimedOutClose() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/worker-frees-scratch-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + try { + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Timed-out close: hands scratch ownership to the worker and + // returns. This is the last close() call this test makes. + manager.setWorkerJoinTimeoutMillis(50L); + manager.close(); + Thread worker = readWorkerThread(manager); + Assert.assertTrue("worker must still be live after the timed-out close", + worker != null && worker.isAlive()); + Assert.assertTrue("scratch must still be allocated while the worker may use it", + readPathScratchImpl(manager) != 0L); + + // Release the pass. running=false already, so the worker + // finishes the pass and exits — its exit block must free the + // handed-over scratch buffer without ANY further caller action. + releaseWorker.countDown(); + worker.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("worker never exited after release", worker.isAlive()); + Assert.assertEquals( + "worker exit block must free the handed-over path scratch — no retried " + + "close() runs in production after a timed-out close, so leaving " + + "the free to a retry leaks the native buffer for the process " + + "lifetime", + 0L, readPathScratchImpl(manager)); + + // Reap the dead thread for tidiness; must not double-free. + manager.close(); + Assert.assertNull("retried close must reap the exited worker", + readWorkerThread(manager)); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.close(); + ring.close(); + } + }); + } + /** * Pins the {@link SegmentManager#deferUntilWorkerExit} handoff contract * that {@code CursorSendEngine.close()}'s slot-ownership transfer depends @@ -473,6 +691,24 @@ private static void cleanupRecursively(String dir) { } } + private static Object readHotSpare(SegmentRing ring) throws Exception { + Field f = SegmentRing.class.getDeclaredField("hotSpare"); + f.setAccessible(true); + return f.get(ring); + } + + private static Object readInServiceRing(SegmentManager manager) throws Exception { + Field inServiceF = SegmentManager.class.getDeclaredField("inService"); + inServiceF.setAccessible(true); + Object entry = inServiceF.get(manager); + if (entry == null) { + return null; + } + Field ringF = entry.getClass().getDeclaredField("ring"); + ringF.setAccessible(true); + return ringF.get(entry); + } + private static long readPathScratchImpl(SegmentManager manager) throws Exception { Field pathScratchF = SegmentManager.class.getDeclaredField("pathScratch"); pathScratchF.setAccessible(true); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 0d4b9f43..015cb0d8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -34,6 +34,7 @@ import java.nio.file.Paths; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -122,6 +123,54 @@ public void testReleaseConfirmsAndIsIdempotent() throws Exception { }); } + /** + * The {@code release() == false} branch: when the OS reports a close + * failure, release must (a) return {@code false} so owners gating a + * "slot reusable" signal never see a lie, (b) RETAIN the fd — forgetting + * it would misreport the lock state and forfeit any later retry — and + * (c) keep returning {@code false} on repeat attempts while the failure + * persists. Once the close succeeds, release confirms and stays + * confirmed. {@code Files.close} cannot be made to fail through the + * public API, so the fd is swapped to a known-bad descriptor and + * restored afterwards. + */ + @Test + public void testFailedCloseRetainsFdAndReportsFalse() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/failed-release"; + SlotLock lock = SlotLock.acquire(slot); + java.lang.reflect.Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(lock); + assertTrue("precondition: acquire must hold a live fd", realFd >= 0); + try { + // A non-negative fd no process has open: close(2) fails EBADF. + fdField.setInt(lock, 1_000_000_000); + assertFalse("release must report false when the OS close fails", + lock.release()); + assertEquals("failed release must retain the fd for a retry — " + + "dropping it would misreport the flock as released", + 1_000_000_000, fdField.getInt(lock)); + assertFalse("repeat release must keep reporting false while the failure persists", + lock.release()); + // While the release is unconfirmed the REAL flock is still + // held — a second acquire on the slot must fail. + try (SlotLock ignored = SlotLock.acquire(slot)) { + fail("slot must not be acquirable while the original flock fd is still open"); + } catch (IllegalStateException expected) { + // good — unconfirmed release really means "still locked". + } + } finally { + fdField.setInt(lock, realFd); + } + assertTrue("release must confirm once the close succeeds", lock.release()); + assertTrue("confirmed release must stay confirmed", lock.release()); + try (SlotLock again = SlotLock.acquire(slot)) { + assertEquals(slot, again.slotDir()); + } + }); + } + @Test public void testTwoDifferentSlotsCoexist() throws Exception { TestUtils.assertMemoryLeak(() -> { From 79e08475ae6418c6cf56df9fedc2f06d90a1dbb4 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 11 Jul 2026 02:15:55 +0100 Subject: [PATCH 12/26] Make final-probe regression deterministic Release the retired slot from a test-only hook after the positive borrow wait has actually exhausted its budget. This removes the scheduler-dependent sleep and guarantees the test reaches the final post-wait probe. --- .../io/questdb/client/impl/SenderPool.java | 16 ++++ .../client/test/impl/SenderPoolSfTest.java | 77 ++++++++----------- 2 files changed, 49 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 34b59191..f11a5912 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -148,6 +148,11 @@ public final class SenderPool implements AutoCloseable { private final Condition slotReleased; // True iff the configuration enables store-and-forward (sf_dir set). private final boolean storeAndForward; + // Test seam: runs after a capacity-starved borrow's condition wait has + // exhausted its positive timeout, before the loop's terminal pass. Null in + // production; regression tests release a retired slot here to prove that + // the terminal pass re-probes returned capacity before throwing. + private volatile Runnable borrowWaitExpiredHook; // Slots removed from `all` whose delegate is still releasing its flock. // They keep reserving capacity (and their slotInUse mark) until the // flock drops, so the cap check and the slot allocator stay consistent @@ -818,6 +823,12 @@ public PooledSender borrow() { } try { remainingNanos = slotReleased.awaitNanos(remainingNanos); + if (remainingNanos <= 0) { + Runnable hook = borrowWaitExpiredHook; + if (hook != null) { + hook.run(); + } + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new LineSenderException("interrupted while waiting for a Sender from the pool"); @@ -830,6 +841,11 @@ public PooledSender borrow() { } } + @TestOnly + public void setBorrowWaitExpiredHook(Runnable hook) { + this.borrowWaitExpiredHook = hook; + } + /** * Raises the shutdown signal early -- without tearing down live delegates -- * so an in-flight startup-recovery step driven on the {@link PoolHousekeeper} diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index abe95d56..a268b14a 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -1290,14 +1290,13 @@ public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Except @Test public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception { - // Release-during-wait twin of the zero-timeout test. A borrower parks - // in awaitNanos while the retired slot's flock is genuinely held; the - // deferred cleanup then releases the flock mid-wait. Nothing signals - // slotReleased (the release is delegate-side, volatile writes only), so - // the borrower sleeps out its full budget. Pre-fix its wake-up pass hit - // the terminal timeout check before reprobeRetiredSlots() and threw -- - // missing capacity that had already come back. Post-fix the wake-up - // pass probes first, recovers the index, and the creation is admitted. + // Positive-timeout twin of the zero-timeout test. A borrower parks in + // awaitNanos while the retired slot's flock is genuinely held and + // sleeps out its full budget. A test hook releases the flock after the + // wait reports expiry but before the terminal loop pass. Pre-fix that + // pass hit the timeout check before reprobeRetiredSlots() and threw -- + // missing capacity that had already come back. Post-fix the terminal + // pass probes first, recovers the index, and admits the creation. TestUtils.assertMemoryLeak(() -> { // Phase 1: strand unacked data under default-0. try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { @@ -1318,8 +1317,9 @@ public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception Assert.assertTrue("unacked data must persist under default-0", hasSegmentFile(slot("default-0"))); - // Phase 2: maxSize=1, generous budget so the mid-wait release lands - // comfortably inside it. + // Phase 2: maxSize=1 and a positive acquire budget. A test hook + // releases the flock only after awaitNanos() has returned with that + // budget exhausted, so the terminal wake-up pass is deterministic. CountingAckHandler handler = new CountingAckHandler(); try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { int ackPort = ack.getPort(); @@ -1340,49 +1340,38 @@ public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception return real; }; - try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 2_000, factory)) { + try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 100, factory)) { Assert.assertNotNull("recovery must have built slot 0", forged.get()); Assert.assertEquals("precondition: startup recovery must retire the slot", 1, pool.leakedSlotCount()); - // Borrower parks: its pre-park probe correctly fails while - // the flock is genuinely held. - AtomicReference failure = new AtomicReference<>(); - AtomicReference borrowed = new AtomicReference<>(); - Thread borrower = new Thread(() -> { + AtomicBoolean waitExpired = new AtomicBoolean(); + pool.setBorrowWaitExpiredHook(() -> { + Assert.assertTrue("expired-wait hook must run exactly once", + waitExpired.compareAndSet(false, true)); + Sender recoverer = forged.get(); try { - borrowed.set(pool.borrow()); - } catch (Throwable e) { - failure.set(e); + setBooleanField(recoverer, "closed", false); + } catch (Exception e) { + throw new RuntimeException(e); } + // The flock drops only after the positive awaitNanos() + // budget is exhausted. This delegate-side release does + // not signal slotReleased. + recoverer.close(); }); - borrower.start(); - - // Let the borrower enter borrow() and park. If a CI stall - // delays it past the release below, the pre-park probe - // recovers instead and the test degrades to a pass -- never - // a flaky failure. - Thread.sleep(300); - - // Mid-wait release: flock drops, no signal reaches the pool. - Sender recoverer = forged.get(); - setBooleanField(recoverer, "closed", false); - recoverer.close(); - - borrower.join(10_000); - Assert.assertFalse("borrower must have finished", borrower.isAlive()); - if (failure.get() != null) { - throw new AssertionError( - "borrower must recover the retired slot on its wake-up pass, not time out", - failure.get()); - } - PooledSender b = borrowed.get(); - Assert.assertNotNull("borrower must have obtained a sender", b); try { - Assert.assertEquals("wake-up probe must recover the retired slot's capacity", - 0, pool.leakedSlotCount()); + PooledSender b = pool.borrow(); + try { + Assert.assertTrue("borrow must exhaust its positive wait budget", + waitExpired.get()); + Assert.assertEquals("wake-up probe must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + } finally { + b.close(); + } } finally { - b.close(); + pool.setBorrowWaitExpiredHook(null); } } } From dd0c5478a27a0fd9c6a1b3f333371a46eec66d86 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 11 Jul 2026 03:32:28 +0100 Subject: [PATCH 13/26] docs(qwp): fix load-bearing concurrency comments that contradict the code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five doc sites documented behavior the code deliberately does not have; each invited a future 'fix' that would reintroduce the hazard the quiescence work eliminated. Comment-only change. - CursorSendEngine.closeCompleted field doc: carve the failed-flock- release case out of the retry sentence. A retried close() exits at the consumed terminalCleanupClaimed CAS and never calls SlotLock.release() again — deliberate, pinned by testUnconfirmedFlockReleaseKeepsCloseIncomplete. - CursorSendEngine.finishClose javadoc: 'must hold the engine monitor' was false for completeDeferredClose (deliberately monitor-free to avoid the join livelock). State the real contract: monitor (close path) OR worker exit path; in all cases the CAS must be won. - CursorSendEngine.isCloseCompleted javadoc: admit the third, unrecoverable state — failed flock release never flips; only process exit frees the flock. - SegmentManager.isWorkerReaped javadoc: the fall-through reap nulls workerThread while the thread may still be running deferred engine cleanups; the engine-side CAS, not this predicate, is the exclusion. - SegmentManager.serviceRing0 trim comment: narrow 'stale snapshots' to mid-pass-deregistered entries — the claim gate makes the trim block unreachable for pre-pass-deregistered entries. - SenderPool reclaimSlot/retireLease: 'retired permanently' contradicted retiredSlots.add + reprobeRetiredSlots recovery three lines down. - QwpWebSocketSender post-guard comment: the incomplete-close branch is not only the owned-manager handoff; document the no-handoff cases (shared manager, failed handoff registration, failed flock release) where the re-probe never flips. --- .../qwp/client/QwpWebSocketSender.java | 22 ++++++++----- .../client/sf/cursor/CursorSendEngine.java | 31 +++++++++++++------ .../qwp/client/sf/cursor/SegmentManager.java | 18 ++++++++--- .../io/questdb/client/impl/SenderPool.java | 15 ++++++--- 4 files changed, 61 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index f1e3533c..268728cc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1272,13 +1272,21 @@ public void close() { ownsCursorEngine = false; slotLockReleased = true; } else { - // The manager worker did not quiesce. Preserve ownership - // and report the retained flock so pools retire this slot. - // Repeated Sender.close() calls remain no-ops by contract. - // Engine cleanup was handed to the worker's exit path - // (owned manager); the getter re-probes the retained - // engine so the pool can reclaim the slot once cleanup - // actually completes. + // Engine close() could not confirm a released flock. + // Preserve ownership and report the retained flock so + // pools retire this slot. Repeated Sender.close() calls + // remain no-ops by contract. Recovery depends on WHY the + // close is incomplete: when cleanup was handed to the + // worker's exit path (owned manager — the only shipping + // configuration), isSlotLockReleased()'s re-probe of the + // retained engine flips once that cleanup completes. + // When there was no handoff, the re-probe never flips + // and the slot stays retired until process exit: a + // shared-manager engine or a failed handoff registration + // needs a retried engine.close() that this one-shot + // sender never issues, and a failed flock release has + // consumed the engine's cleanup claim, so nothing can + // re-run the release. slotLockReleased = false; retainedEngine = engine; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index c9b85aea..42ca4d17 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -124,9 +124,16 @@ public final class CursorSendEngine implements QuietCloseable { // SenderPool.reprobeRetiredSlots), so publishing before the release // would let a replacement engine's SlotLock.acquire collide with the // still-open fd. Stays false when a close attempt could not confirm - // manager-worker quiescence (or the flock release itself failed) and had - // to leak the ring/watermark/slot lock — in that case a later close() - // call retries the cleanup (the worker may have exited by then). + // manager-worker quiescence and had to leak the ring/watermark/slot + // lock — in that case a later close() call retries the cleanup (the + // worker may have exited by then). Also stays false — permanently — + // when finishClose() ran but the flock release itself failed: the + // terminalCleanupClaimed CAS is consumed, so a retried close() exits at + // the CAS without re-running the cleanup or calling SlotLock.release() + // again. That non-retry is deliberate (a retry would double-close the + // ring/watermark and double-release the flock; pinned by + // testUnconfirmedFlockReleaseKeepsCloseIncomplete) — the kernel drops + // the flock at process exit. // volatile: latched by finishClose(), but read lock-free by // isCloseCompleted() from pool threads re-probing a retired slot (see the // getter for why it must not synchronize). @@ -674,7 +681,10 @@ public synchronized void close() { * {@code closeCompleted} (via {@code isSlotLockReleased()}), so it must * never be visible while the flock fd is still open, or a replacement * engine races the release and fails acquisition on a live slot. - * The caller must hold the engine monitor and + * The caller must either hold the engine monitor (the close() path) or + * run on the manager worker's exit path ({@link #completeDeferredClose}, + * which deliberately takes no monitor — adding one would recreate the + * join livelock the CAS exists to avoid); in all cases the caller * must have established that the manager worker can no longer touch the * slot directory (reaped, provably exited, or running this on its own * exit path) AND have won the {@link #terminalCleanupClaimed} CAS — the @@ -796,11 +806,14 @@ private void completeDeferredClose(boolean fullyDrained) { * confirmed release of the SF slot lock — the flip is published * strictly after the flock fd is closed, so observing {@code true} * guarantees the slot dir is acquirable by a replacement engine. A false - * value after close means manager-worker quiescence could not be - * confirmed (or the flock release itself failed) and the - * worker-reachable resources were retained deliberately — either handed to the worker's exit path (owned manager), - * which flips this to true the moment the worker's in-flight pass - * finishes, or leaked until a retried close() (shared manager). Owners + * value after close means the worker-reachable resources were retained + * deliberately, for one of three reasons: cleanup was handed to the + * worker's exit path (owned manager), which flips this to true the + * moment the worker's in-flight pass finishes; cleanup was leaked until + * a retried close() (shared manager, or a failed handoff registration); + * or the flock release itself failed — that last case never flips: the + * {@link #terminalCleanupClaimed} CAS is consumed, so no retry re-runs + * the release, and only process exit frees the flock. Owners * must not reuse the slot while this is false; pools may re-probe it to * recover a retired slot's capacity once it flips. *

    diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 51bd204e..24a9353a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -376,9 +376,16 @@ public boolean awaitRingQuiescence(SegmentRing ring) { } /** - * True when no manager worker thread can be running: either - * {@link #start()} was never called, or a {@link #close()} confirmed - * worker termination and reaped the thread. Owners use this as a + * True when the manager worker can no longer run a service pass: either + * {@link #start()} was never called, or a {@link #close()} reaped the + * thread — including the fall-through reap that observes + * {@code workerLoopExited} while the thread is still alive. In that case + * the worker may still be RUNNING its deferred engine cleanups + * (see {@link #deferUntilWorkerExit}: munmap/unlink/flock-release on the + * exit path) when this flips true, so this predicate is NOT the + * exclusion between a retried engine close() and the worker's deferred + * cleanup — the engine-side {@code terminalCleanupClaimed} CAS is. + * Owners use this as a * stronger fallback barrier when {@link #awaitRingQuiescence(SegmentRing)} * times out but a subsequent {@code close()} join succeeded. */ @@ -742,7 +749,10 @@ private void serviceRing0(RingEntry e) { // The watermark write and totalBytes commit are registration-gated // under `lock` so stale worker snapshots cannot touch the // engine-owned watermark or mutate accounting after deregister() - // returns. drainTrimmable still runs for stale snapshots: it + // returns. drainTrimmable still runs for entries deregistered + // MID-pass (entries deregistered before the pass started never + // get here — the registration/in-service claim at the top of + // serviceRing skips them entirely): it // transfers ownership of fully-acked sealed segments to this // worker, preserving the old close + unlink behavior. // diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index f11a5912..9e7ecd35 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1062,7 +1062,9 @@ private void retireLease(PooledSender ps, String context) { pendingLeaseTeardowns--; if (reserved) { // Free the index only when the flock was released; a slot - // left locked is retired permanently. + // left locked is retired into retiredSlots, recoverable + // by reprobeRetiredSlots() if the deferred cleanup drops + // the flock later. reclaimSlot(s, context); } slotReleased.signalAll(); @@ -1350,10 +1352,13 @@ private static boolean flockReleased(SenderSlot s) { * Reclaims one SF slot after its delegate's {@code close()} has been * attempted. When the flock was released the index returns to the free * set; when {@code close()} returned with the flock still held because an - * I/O or manager worker did not stop, the slot is retired permanently -- - * {@code leakedSlots++} and {@code slotInUse[idx]} stays set -- so the cap - * math accounts for the lost capacity and no later borrow ever reuses the - * still-locked dir. Either way {@code closingSlots} is decremented. + * I/O or manager worker did not stop, the slot is retired -- + * {@code leakedSlots++}, {@code slotInUse[idx]} stays set, and the sender + * joins {@code retiredSlots} -- so the cap math accounts for the lost + * capacity and no borrow reuses the still-locked dir unless + * {@link #reprobeRetiredSlots} later observes the deferred cleanup's + * release and recovers the index. Either way {@code closingSlots} is + * decremented. *

    * Caller must hold {@code lock} and is responsible for signalling waiters * (only the free path admits a new creation). Shared by From b89384fef4a28bb34a8d3c1e36dba5d8ef60a4a9 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 11 Jul 2026 14:53:55 +0100 Subject: [PATCH 14/26] fix(qwp): recover SF slots after deferred cleanup --- core/src/main/c/share/files.c | 12 + core/src/main/c/windows/files.c | 14 + .../qwp/client/QwpWebSocketSender.java | 42 +- .../client/sf/cursor/CursorSendEngine.java | 231 +++++--- .../qwp/client/sf/cursor/SegmentManager.java | 251 +++++++-- .../qwp/client/sf/cursor/SlotLock.java | 37 +- .../io/questdb/client/impl/SenderPool.java | 24 +- .../cutlass/qwp/client/CloseDrainTest.java | 52 ++ ...ocketSenderCursorEngineAttachmentTest.java | 141 +++++ .../client/SlotLockReleasedContractTest.java | 27 +- ...CursorSendEngineSlotReacquisitionTest.java | 23 +- .../sf/cursor/CursorSendEngineTest.java | 19 + ...gineClosePublishAfterFlockReleaseTest.java | 48 +- .../SegmentManagerPassBarrierBenchmark.java | 131 +++++ .../qwp/client/sf/cursor/SlotLockTest.java | 36 +- .../client/test/impl/SenderPoolSfTest.java | 533 +++++++++++++++++- 16 files changed, 1387 insertions(+), 234 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java diff --git a/core/src/main/c/share/files.c b/core/src/main/c/share/files.c index 75f228fa..209c4490 100644 --- a/core/src/main/c/share/files.c +++ b/core/src/main/c/share/files.c @@ -236,6 +236,18 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_lock return flock((int) fd, LOCK_EX | LOCK_NB); } +JNIEXPORT jint JNICALL Java_io_questdb_client_cutlass_qwp_client_sf_cursor_SlotLock_release0 + (JNIEnv *e, jclass cl, jint fd) { + if (flock((int) fd, LOCK_UN) != 0) { + return -1; + } + /* Unlock success confirms that the slot is reusable. close() is one-shot: + * POSIX leaves descriptor state unspecified on EINTR, so retrying its + * numeric value could close an unrelated descriptor after reuse. */ + (void) close((int) fd); + return 0; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mkdir0 (JNIEnv *e, jclass cl, jlong lpszPath, jint mode) { return mkdir((const char *) (uintptr_t) lpszPath, (mode_t) mode); diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index 119f6315..d1d3b413 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -331,6 +331,20 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_lock return 0; } +JNIEXPORT jint JNICALL Java_io_questdb_client_cutlass_qwp_client_sf_cursor_SlotLock_release0 + (JNIEnv *e, jclass cl, jint fd) { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (!UnlockFileEx(FD_TO_HANDLE(fd), 0, MAXDWORD, MAXDWORD, &ov)) { + SaveLastError(); + return -1; + } + /* Unlock success confirms that the slot is reusable. Match POSIX by + * closing once without making handle cleanup part of that signal. */ + (void) CloseHandle(FD_TO_HANDLE(fd)); + return 0; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mkdir0 (JNIEnv *e, jclass cl, jlong lpszPath, jint mode) { (void) mode; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index f1e3533c..3586a4d2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -194,6 +194,9 @@ public class QwpWebSocketSender implements Sender { // up to this many millis for ackedFsn to catch up to publishedFsn. private long closeFlushTimeoutMillis = 5_000L; private volatile boolean closed; + // Test-only lifecycle witness. close() invokes and clears it strictly after + // publishing closed=true and before starting any drain or teardown work. + private volatile Runnable closeStartedHook; private boolean connected; private SenderConnectionDispatcher connectionDispatcher; // Async-delivery sink for SenderConnectionEvent notifications. Default @@ -1057,6 +1060,16 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { public void close() { if (!closed) { closed = true; + Runnable hook = closeStartedHook; + closeStartedHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable t) { + // A test witness must never prevent production resource cleanup. + LOG.error("Error in close-started test hook: {}", String.valueOf(t)); + } + } boolean ioThreadStopped = true; // Captures the first error from the flush/drain path AND any // secondary errors from cleanup steps (added via addSuppressed). @@ -1275,10 +1288,10 @@ public void close() { // The manager worker did not quiesce. Preserve ownership // and report the retained flock so pools retire this slot. // Repeated Sender.close() calls remain no-ops by contract. - // Engine cleanup was handed to the worker's exit path - // (owned manager); the getter re-probes the retained - // engine so the pool can reclaim the slot once cleanup - // actually completes. + // Engine cleanup was handed to a safe manager-worker path: + // owned-manager exit or shared-manager ring-pass completion. + // The getter re-probes the retained engine so the pool can + // reclaim the slot once cleanup actually completes. slotLockReleased = false; retainedEngine = engine; } @@ -1335,8 +1348,8 @@ public void close() { * index reserved instead of reusing the still-locked dir. *

    * Not a one-shot snapshot: when close() left engine cleanup pending on a - * worker/I/O-thread exit path, this re-probes the retained engine and - * latches true the moment that cleanup completes — pools re-probe retired + * manager-worker quiescence or I/O-thread exit path, this re-probes the + * retained engine and latches true the moment that cleanup completes — pools re-probe retired * slots through this getter to recover their capacity. Monotonic: * false→true only, never back. Lock-free (volatile reads) so pools may * call it under their capacity lock. @@ -2194,6 +2207,16 @@ public void setClientFactoryOverride(java.util.function.Supplier completeDeferredClose(fullyDrained)); + handedOff = manager.deferOwnedEngineCloseUntilWorkerExit(deferredClose); } catch (Throwable ignored) { - // Allocation failure (OOM building the cleanup lambda or - // growing the manager's exitCleanups list). Unlike the exact - // false return below, a throw carries NO worker-liveness - // information — the two must never be conflated, or the - // inline cleanup below would release worker-reachable - // resources under a possibly-live worker. + // Unexpected monitor/VM failure carries no worker-liveness + // information. Ordinary handoff cannot allocate: both the + // callback and the manager's single slot were preallocated. registrationFailed = true; } if (handedOff) { @@ -645,22 +670,41 @@ public synchronized void close() { workerQuiescent = true; } if (!workerQuiescent) { - // Shared (caller-owned) manager: its worker keeps serving other - // rings indefinitely, so there is no exit path to hand cleanup - // to — leak and let a retried close() reclaim once the pass ends. - LOG.error("SF manager worker did not quiesce during engine close; leaking the " - + "ring, watermark and slot lock so a stale worker cannot corrupt a " - + "future engine on slot {}. The kernel releases the slot flock on " - + "process exit; close() may be invoked again to retry cleanup once " - + "the worker has exited.", sfDir == null ? "" : sfDir); - return; + // A shared manager keeps serving sibling rings, so it cannot use + // the whole-worker exit handoff above. Transfer cleanup to this + // ring's current pass instead. Registration and pass completion + // share the manager lock: true means the worker owns cleanup; + // false means the pass already finished and inline cleanup is safe. + boolean handedOff = false; + boolean registrationFailed = false; + try { + handedOff = manager.deferUntilRingQuiescent(ring, deferredClose); + } catch (Throwable ignored) { + registrationFailed = true; + } + if (handedOff) { + LOG.error("SF manager worker did not quiesce during engine close; ring, watermark " + + "and slot-lock release are handed to the current ring pass and run the " + + "moment that pass finishes. The slot stays locked (and " + + "isCloseCompleted() false) until then, so no replacement engine can " + + "race the stale worker on slot {}", + sfDir == null ? "" : sfDir); + return; + } + if (registrationFailed) { + LOG.error("SF ring-pass cleanup handoff registration failed during engine close; " + + "leaking the ring, watermark and slot lock so a possibly-live worker " + + "cannot corrupt a future engine on slot {}. The kernel releases the " + + "flock on process exit.", sfDir == null ? "" : sfDir); + return; + } + workerQuiescent = true; } if (!terminalCleanupClaimed.compareAndSet(false, true)) { - // A worker-exit handoff from an earlier close() attempt owns the - // terminal cleanup: it has run or is finishing right now on the - // exiting worker. closeCompleted flips the moment it is done — - // owners observe it via isCloseCompleted(), never by re-running - // the cleanup here (that would double-release ring/flock). + // A worker-exit handoff or earlier close owns the one-time + // ring/watermark cleanup. Once that work is published complete, + // this close may still retry an unconfirmed flock release. + retryFlockReleaseIfReady(); return; } finishClose(fullyDrained); @@ -672,7 +716,7 @@ public synchronized void close() { * is confirmed — latches {@link #closeCompleted}. Publish order * is load-bearing: pools free the slot index the instant they observe * {@code closeCompleted} (via {@code isSlotLockReleased()}), so it must - * never be visible while the flock fd is still open, or a replacement + * never be visible while the flock is still held, or a replacement * engine races the release and fails acquisition on a live slot. * The caller must hold the engine monitor and * must have established that the manager worker can no longer touch the @@ -718,13 +762,12 @@ private void finishClose(boolean fullyDrained) { // releasing the flock is safe even if a step above threw. Leaking // it would strand the slot until process exit for no reason. // - // ORDER MATTERS: release the flock FIRST, verify it, and only - // then publish closeCompleted. Pools read isCloseCompleted() as - // "the slot dir is reusable" and free the slot index the moment - // it flips; publishing before Files.close(fd) completes would - // open a window where a replacement engine's SlotLock.acquire - // collides with the still-open fd and fails with a spurious - // "slot already in use". + // ORDER MATTERS: explicitly release the flock, verify it, and + // only then publish closeCompleted. Pools read isCloseCompleted() + // as "the slot dir is reusable" and free the slot index the moment + // it flips. SlotLock closes the fd once after confirmed unlock, + // but that close result cannot safely control publication because + // POSIX may consume the fd even when close reports failure. Runnable hook = beforeFlockReleaseHook; if (hook != null) { try { @@ -733,32 +776,21 @@ private void finishClose(boolean fullyDrained) { // test-only; must never block the release } } - boolean released; - if (slotLock != null) { - try { - released = slotLock.release(); - } catch (Throwable ignored) { - released = false; - } - } else { - released = true; - } - if (released) { - closeCompleted = true; - } else { - // Unconfirmed release: the flock may still be held, so keep - // closeCompleted false — the owning pool keeps the slot - // retired (leakedSlots accounting) instead of reusing a - // possibly-locked dir. The kernel releases the flock on - // process exit regardless. - LOG.error("SF slot flock release failed during engine close; keeping " - + "closeCompleted=false so the pool cannot reuse a possibly " - + "still-locked slot dir; the kernel releases the flock on " - + "process exit [slot={}]", sfDir == null ? "" : sfDir); - } + terminalResourcesCleaned = true; + retryFlockReleaseIfReady(); } } + /** + * Installs a constructor fault hook immediately before the bound deferred + * close callback is created. Test-only: proves callback allocation failure + * occurs before an owned manager acquires native resources. + */ + @TestOnly + public static void setBeforeDeferredCloseCreationHook(Runnable hook) { + beforeDeferredCloseCreationHook = hook; + } + /** * Installs a hook that {@link #finishClose} runs between the terminal * cleanup and the slot-flock release. Test-only: makes the otherwise @@ -772,29 +804,94 @@ public void setBeforeFlockReleaseHook(Runnable hook) { } /** - * Runs on the manager worker's exit path when {@link #close()} handed - * cleanup ownership to the worker (see {@code deferUntilWorkerExit}). + * Runs on a safe manager-worker handoff path when {@link #close()} moved + * cleanup ownership to either a shared manager's ring-pass completion or + * an owned manager's worker exit. * Deliberately does NOT take the engine monitor: a retried close() can * hold it while joining this very worker, and the thread cannot die until * this method returns — the {@link #terminalCleanupClaimed} CAS provides * the exactly-once exclusion instead, so the worker always exits promptly * and the racing close() converges via {@code isWorkerReaped()}. */ - private void completeDeferredClose(boolean fullyDrained) { + private void completeDeferredClose() { if (!terminalCleanupClaimed.compareAndSet(false, true)) { // A retried close() (or an earlier duplicate handoff) already ran // the terminal cleanup. return; } - finishClose(fullyDrained); - LOG.info("deferred SF engine cleanup completed on manager-worker exit; slot released: {}", - sfDir == null ? "" : sfDir); + finishClose(fullyDrainedForDeferredClose); + LOG.info("deferred SF engine resource cleanup completed after manager-worker quiescence; " + + "slot release confirmed: {} [slot={}]", + closeCompleted, sfDir == null ? "" : sfDir); + } + + private Runnable createDeferredClose() { + Runnable hook = beforeDeferredCloseCreationHook; + if (hook != null) { + hook.run(); + } + return this::completeDeferredClose; + } + + private boolean retryFlockReleaseIfReady() { + if (closeCompleted || !terminalResourcesCleaned) { + return closeCompleted; + } + boolean released; + if (slotLock != null) { + try { + released = slotLock.release(); + } catch (Throwable ignored) { + released = false; + } + } else { + released = true; + } + if (released) { + closeCompleted = true; + return true; + } + startFlockReleaseRetry(); + return false; + } + + private void startFlockReleaseRetry() { + if (!flockReleaseRetryStarted.compareAndSet(false, true)) { + return; + } + try { + Thread retryThread = new Thread(() -> { + while (!retryFlockReleaseIfReady()) { + try { + Thread.sleep(100L); + } catch (InterruptedException ignored) { + // A failed flock release must remain retryable until + // confirmed or process exit; interruption is not a + // safe reason to abandon the slot permanently. + } + } + }, "qdb-sf-flock-release-retry"); + retryThread.setDaemon(true); + retryThread.start(); + LOG.error("SF slot flock release failed during engine close; keeping " + + "closeCompleted=false and retrying outside the pool lock so " + + "retired capacity recovers after the transient failure [slot={}]", + sfDir == null ? "" : sfDir); + } catch (Throwable t) { + // A later explicit close() can still retry without repeating the + // one-time ring/watermark cleanup. The kernel remains the final + // process-exit backstop if thread creation and all retries fail. + flockReleaseRetryStarted.set(false); + LOG.error("Could not start SF flock-release retry driver; close() may be " + + "invoked again to retry [slot={}, error={}]", + sfDir == null ? "" : sfDir, String.valueOf(t)); + } } /** * Whether {@link #close()} completed all cleanup, including a * confirmed release of the SF slot lock — the flip is published - * strictly after the flock fd is closed, so observing {@code true} + * strictly after explicit unlock succeeds, so observing {@code true} * guarantees the slot dir is acquirable by a replacement engine. A false * value after close means manager-worker quiescence could not be * confirmed (or the flock release itself failed) and the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 51bd204e..15d5c607 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -34,7 +34,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.LockSupport; /** @@ -66,6 +68,14 @@ public final class SegmentManager implements QuietCloseable { public static final long DEFAULT_POLL_NANOS = 1_000_000L; // 1 ms public static final long DISK_FULL_LOG_THROTTLE_NANOS = 30_000_000_000L; // 30 s public static final long UNLIMITED_TOTAL_BYTES = Long.MAX_VALUE; + private static final int ENTRY_DEREGISTERED = 3; + private static final int ENTRY_DEREGISTERED_IN_SERVICE = 2; + private static final int ENTRY_IN_SERVICE = 1; + private static final int ENTRY_REGISTERED = 0; + private static final AtomicReferenceFieldUpdater ENTRY_CLEANUP_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(RingEntry.class, Runnable.class, "quiescenceCleanup"); + private static final AtomicIntegerFieldUpdater ENTRY_STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(RingEntry.class, "state"); private static final Logger LOG = LoggerFactory.getLogger(SegmentManager.class); private static final long WORKER_JOIN_TIMEOUT_MILLIS = 5_000L; @@ -86,6 +96,17 @@ public final class SegmentManager implements QuietCloseable { private final ObjList ringSnapshot = new ObjList<>(); private final ObjList rings = new ObjList<>(); private final long segmentSizeBytes; + // Test seam: runs after a deferred ring-pass cleanup returns. Null in + // production; public sender/pool lifecycle tests use it to observe exact + // callback completion without sleeps or polling. + private volatile Runnable afterRingCleanupHook; + // Test seam: runs at the top of deferUntilWorkerExit, before the + // worker-liveness check. Null in production; registration-failure tests + // throw from it to simulate an allocation failure (OOM building the + // cleanup lambda or growing exitCleanups) while the worker is still + // live. Callers must treat such a throw as "worker state unknown", + // never as the exact false return meaning the worker loop has exited. + private volatile Runnable beforeExitCleanupRegistrationHook; // Test seam: runs on the worker thread just before the install path's // synchronized(lock) entry (the one that performs installHotSpare + the // totalBytes += segmentSize commit). Null in production; tests use it to @@ -97,25 +118,19 @@ public final class SegmentManager implements QuietCloseable { // production; owned-engine close tests use it to prove they take only the // stronger whole-manager join path, not two sequential timeout budgets. private volatile Runnable beforeRingQuiescenceAwaitHook; - // Test seam: runs at the top of deferUntilWorkerExit, before the - // worker-liveness check. Null in production; registration-failure tests - // throw from it to simulate an allocation failure (OOM building the - // cleanup lambda or growing exitCleanups) while the worker is still - // live. Callers must treat such a throw as "worker state unknown", - // never as the exact false return meaning the worker loop has exited. - private volatile Runnable beforeExitCleanupRegistrationHook; // Test seam: runs on the worker thread just before the trim block's // synchronized(lock) entry. Null in production; only // SegmentManagerTrimDeregisterRaceTest installs it, to deterministically // inject a deregister(ring) call into the exact race window that the - // registered flag check inside the trim block closes for watermark writes - // and totalBytes accounting. + // entry-state check inside the trim block closes for watermark writes and + // totalBytes accounting. private volatile Runnable beforeTrimSyncHook; - // Entry currently being serviced by the worker thread, or null when the - // worker is between service passes (or not running). Guarded by - // {@link #lock}; cleared with lock.notifyAll() so awaitRingQuiescence can - // block until an in-flight pass for a just-deregistered ring finishes. - private RingEntry inService; + // Entry the worker is claiming or servicing, or null between passes. + // Volatile publication lets teardown find the entry after deregister() + // removes it from rings. RingEntry.state is the authoritative barrier: + // the worker publishes this reference before its REGISTERED->IN_SERVICE + // CAS and only touches ring resources after that CAS succeeds. + private volatile RingEntry inService; // Cleanup actions handed to the worker's exit block by an owning engine // whose close() found the worker still mid service pass after the bounded // join timed out (see deferUntilWorkerExit). Guarded by {@link #lock}; @@ -124,14 +139,11 @@ public final class SegmentManager implements QuietCloseable { // and no caller-facing lock may ever be held while running third-party // cleanup code. private ObjList exitCleanups; - // Number of threads currently parked in awaitRingQuiescence()'s wait - // loop. Guarded by {@link #lock}. The worker's per-pass finally block - // only calls lock.notifyAll() when this is > 0, so the steady-state - // production path (no quiescence barrier in flight) never notifies. - // Both sides mutate/check under the same lock, so there is no lost - // wakeup: a waiter that increments after the worker's check re-reads - // inService before waiting and sees the pass already finished. - private int quiescenceWaiters; + // A private manager belongs to exactly one CursorSendEngine. Its callback + // is preallocated by that engine and stored directly here, so the critical + // timed-out-close handoff never allocates an ObjList or grows a backing + // array. Guarded by lock and consumed on worker-loop exit outside lock. + private Runnable ownedEngineExitCleanup; private long lastDiskFullLogNs; private volatile boolean running; // pathScratch free-exactly-once coordination between a timed-out close() @@ -273,6 +285,62 @@ public synchronized void close() { } } + /** + * Hands the single owning engine's preallocated close callback to the + * worker exit block. Registration only assigns a reference under + * {@link #lock}; it cannot allocate in the timed-out teardown path. + * Returns {@code false} only after the worker loop has exited (or when it + * never started), so the caller may then clean up inline safely. + */ + public boolean deferOwnedEngineCloseUntilWorkerExit(Runnable cleanup) { + synchronized (lock) { + if (workerLoopExited || workerThread == null) { + return false; + } + if (ownedEngineExitCleanup != null && ownedEngineExitCleanup != cleanup) { + throw new IllegalStateException("owned manager already has an engine-exit cleanup"); + } + ownedEngineExitCleanup = cleanup; + return true; + } + } + + /** + * Hands a cleanup action to the current service pass for {@code ring}. + * The worker runs it outside the manager lock immediately after that pass + * finishes. Returns {@code false} when no pass for the ring remains in + * flight, in which case the caller already owns a quiescent ring and must + * run the cleanup itself. + *

    + * Registration and pass completion coordinate through the entry's atomic + * state and callback fields, so there is no gap without an owner: either + * this method attaches the cleanup while the pass remains active, the + * worker claims an attached cleanup while completing, or this method + * observes completed state and rejects the handoff. A repeated registration + * for the same pass returns {@code true} without replacing its existing + * owner. + */ + public boolean deferUntilRingQuiescent(SegmentRing ring, Runnable cleanup) { + RingEntry e = inService; + if (e == null || e.ring != ring || !e.isInService()) { + return false; + } + Runnable existing = ENTRY_CLEANUP_UPDATER.get(e); + if (existing == null && ENTRY_CLEANUP_UPDATER.compareAndSet(e, null, cleanup)) { + if (e.isInService()) { + return true; + } + // Completion changed the state before observing the callback. + // Remove our callback and clean inline, unless the worker already + // claimed it (in which case that worker remains the owner). + return !ENTRY_CLEANUP_UPDATER.compareAndSet(e, cleanup, null); + } + // A callback already attached to this pass is an owner. The sole + // production caller always supplies the engine's preallocated bound + // callback; duplicates intentionally do not replace it. + return true; + } + /** * Hands a cleanup action to the worker thread's exit block, to run * strictly after the worker loop has finished its final service pass -- @@ -347,10 +415,14 @@ public boolean awaitRingQuiescence(SegmentRing ring) { long deadlineNanos = System.nanoTime() + workerJoinTimeoutMillis * 1_000_000L; boolean interrupted = Thread.interrupted(); try { + RingEntry e = inService; + if (e == null || e.ring != ring || !e.isInService()) { + return true; + } synchronized (lock) { - quiescenceWaiters++; + e.quiescenceWaiters++; try { - while (inService != null && inService.ring == ring) { + while (e.isInService()) { long remainingNanos = deadlineNanos - System.nanoTime(); if (remainingNanos <= 0) { return false; @@ -364,7 +436,7 @@ public boolean awaitRingQuiescence(SegmentRing ring) { } } } finally { - quiescenceWaiters--; + e.quiescenceWaiters--; } } return true; @@ -411,7 +483,7 @@ public void deregister(SegmentRing ring) { // single subtraction covers both the initial seed // and the net manager activity (provisions minus // trims) for this ring. - e.registered = false; + e.deregister(); totalBytes -= ring.totalSegmentBytes(); rings.remove(i); return; @@ -482,6 +554,11 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { wakeWorker(); } + @TestOnly + public void setAfterRingCleanupHook(Runnable hook) { + this.afterRingCleanupHook = hook; + } + @TestOnly public void setBeforeExitCleanupRegistrationHook(Runnable hook) { this.beforeExitCleanupRegistrationHook = hook; @@ -607,33 +684,42 @@ private String nextSparePath(String dir) { } private void serviceRing(RingEntry e) { - // Claim the entry as in-service so deregister-side quiescence - // barriers (awaitRingQuiescence) can wait for this pass to finish. - // A stale snapshot entry deregistered before the pass starts is - // skipped entirely: the deregistering thread may already be - // releasing the ring / watermark / slot resources, so the worker - // must not touch them at all. The registered check and the - // in-service claim are atomic under `lock` — deregister flips - // `registered` under the same lock, so it either prevents this - // pass or the barrier observes it via `inService`. - synchronized (lock) { - if (!e.registered) { - return; - } - inService = e; + // Publish before the CAS so deregister + await can always find the + // entry. The state CAS is the ownership decision: if deregister won, + // this stale snapshot pass skips the ring without touching it. + inService = e; + if (!ENTRY_STATE_UPDATER.compareAndSet(e, ENTRY_REGISTERED, ENTRY_IN_SERVICE)) { + inService = null; + return; } try { serviceRing0(e); } finally { - synchronized (lock) { - inService = null; - // Wake quiescence barriers only when one is actually parked. - // In production no engine ever waits here (owned-manager - // close joins the worker thread instead), so the per-tick - // pass must not pay an unconditional notifyAll. notifyAll, - // not notify: distinct waiters may await different rings. - if (quiescenceWaiters > 0) { - lock.notifyAll(); + e.finishService(); + Runnable cleanup = e.quiescenceCleanup == null + ? null + : ENTRY_CLEANUP_UPDATER.getAndSet(e, null); + inService = null; + // A normal pass performs only the two state CAS operations above. + // The manager monitor is entered here only for an actual close + // waiter; the recheck under lock prevents lost wakeups. + if (e.quiescenceWaiters > 0) { + synchronized (lock) { + if (e.quiescenceWaiters > 0) { + lock.notifyAll(); + } + } + } + if (cleanup != null) { + try { + cleanup.run(); + } catch (Throwable t) { + LOG.error("deferred engine cleanup failed after manager-worker ring pass", t); + } finally { + Runnable hook = afterRingCleanupHook; + if (hook != null) { + hook.run(); + } } } } @@ -704,7 +790,7 @@ private void serviceRing0(RingEntry e) { // observe the spare in the ring (and subtract it) or // run before installation (so no install happens). synchronized (lock) { - if (e.registered) { + if (e.isRegistered()) { e.ring.installHotSpare(spare); totalBytes += segmentSizeBytes; installed = true; @@ -754,7 +840,7 @@ private void serviceRing0(RingEntry e) { hook.run(); } synchronized (lock) { - boolean registered = e.registered; + boolean registered = e.isRegistered(); // Persist the current ackedFsn watermark BEFORE the trim runs. // On host crash between the persist and the unlinks below, the // segments survive and the watermark is correct. On crash AFTER @@ -827,6 +913,7 @@ private void workerLoop() { // here reclaims the native buffer even when the worker outlives // every close() attempt — nobody else retries manager cleanup. ObjList cleanups; + Runnable ownedEngineCleanup; synchronized (lock) { workerLoopExited = true; if (scratchHandedToWorker && !scratchFreed) { @@ -835,8 +922,10 @@ private void workerLoop() { } cleanups = exitCleanups; exitCleanups = null; + ownedEngineCleanup = ownedEngineExitCleanup; + ownedEngineExitCleanup = null; } - // Deferred engine cleanups (see deferUntilWorkerExit) run OUTSIDE + // Deferred engine cleanups run OUTSIDE // `lock`: they perform syscalls (munmap, unlink, flock release) // and must never execute under a lock that close()/register/ // deregister callers contend on. Running them after the loop body @@ -845,6 +934,13 @@ private void workerLoop() { // monitor — a retried engine.close() joins this thread while // holding the engine monitor, which is why the engine side uses a // lock-free claim (CAS), not synchronization, for exactly-once. + if (ownedEngineCleanup != null) { + try { + ownedEngineCleanup.run(); + } catch (Throwable t) { + LOG.error("deferred owned-engine cleanup failed on manager-worker exit", t); + } + } if (cleanups != null) { for (int i = 0, n = cleanups.size(); i < n; i++) { try { @@ -870,16 +966,57 @@ private static final class RingEntry { // Survives across multiple serviceRing ticks and avoids a // write-storm when ackedFsn is steady. long lastPersistedAck = -1L; - // Guarded by SegmentManager.lock. A worker snapshot may retain this - // entry after deregister() removes it from rings; registered=false is - // the O(1) ownership check that prevents post-deregister writes through - // the engine-owned watermark, hot-spare installs, and accounting. - boolean registered = true; + // Updated lock-free by deferUntilRingQuiescent and pass completion. + // The field updater avoids one AtomicReference allocation per ring. + volatile Runnable quiescenceCleanup; + // Waiters mutate this under SegmentManager.lock; pass completion reads + // it before taking the otherwise-cold notification path. + volatile int quiescenceWaiters; + // REGISTERED -> IN_SERVICE -> REGISTERED on a normal pass. + // Deregistration changes either registered state to its corresponding + // deregistered state; completion then changes DEREGISTERED_IN_SERVICE + // to DEREGISTERED. The field updater avoids per-entry allocation. + volatile int state = ENTRY_REGISTERED; RingEntry(SegmentRing ring, String dir, AckWatermark watermark) { this.ring = ring; this.dir = dir; this.watermark = watermark; } + + void deregister() { + while (true) { + int current = state; + int next = current == ENTRY_IN_SERVICE + ? ENTRY_DEREGISTERED_IN_SERVICE + : ENTRY_DEREGISTERED; + if (current == ENTRY_DEREGISTERED || current == ENTRY_DEREGISTERED_IN_SERVICE + || ENTRY_STATE_UPDATER.compareAndSet(this, current, next)) { + return; + } + } + } + + void finishService() { + while (true) { + int current = state; + int next = current == ENTRY_DEREGISTERED_IN_SERVICE + ? ENTRY_DEREGISTERED + : ENTRY_REGISTERED; + if (ENTRY_STATE_UPDATER.compareAndSet(this, current, next)) { + return; + } + } + } + + boolean isInService() { + int current = state; + return current == ENTRY_IN_SERVICE || current == ENTRY_DEREGISTERED_IN_SERVICE; + } + + boolean isRegistered() { + int current = state; + return current == ENTRY_REGISTERED || current == ENTRY_IN_SERVICE; + } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 1b72d157..2bbcc109 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -36,9 +36,9 @@ * Advisory exclusive lock for a single SF slot directory. *

    * One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx} - * for the entire lifetime of the engine that owns the slot. The lock is - * automatically released when the fd is closed — including on hard process - * exit, since the kernel cleans up file locks for terminated processes. + * for the entire lifetime of the engine that owns the slot. Normal teardown + * explicitly unlocks it before closing the fd; hard process exit remains a + * backstop because the kernel cleans up file locks for terminated processes. *

    * The holder's PID is written to a sibling {@code .lock.pid} file at * acquisition time. A failed acquisition reads it back so the error message @@ -117,31 +117,32 @@ public String slotDir() { } /** - * Releases the flock by closing the lock fd and reports whether the - * release was confirmed. Closing the fd releases the lock. We do - * NOT remove the {@code .lock} file or the {@code .lock.pid} sidecar — - * a stale PID is harmless (next acquirer overwrites {@code .lock.pid} - * on success). + * Explicitly releases the flock and reports whether the release was + * confirmed. After a successful unlock the native primitive closes + * the descriptor once, best-effort, and this object forgets its numeric + * value. It never retries that close: POSIX leaves descriptor state + * unspecified after some close failures (notably {@code EINTR}), so a + * retry could close an unrelated descriptor that reused the same number. + * We do NOT remove the {@code .lock} file or {@code .lock.pid} sidecar; a + * stale PID is harmless because the next acquirer overwrites it. *

    - * On close failure the fd is retained: the flock may still be - * held by this process, so forgetting the fd would misreport the lock - * state and forfeit any chance of a later retry. Idempotent — once - * released, subsequent calls return {@code true}. + * When the explicit unlock itself fails, the fd is retained so a later + * attempt can safely retry the non-consuming unlock operation. Idempotent + * once the unlock has succeeded. *

    * Owners that gate a "slot dir is reusable" signal on the release * (e.g. {@code CursorSendEngine.finishClose} publishing * {@code closeCompleted}) must call this and check the result rather * than {@link #close()}, which is best-effort by contract. * - * @return {@code true} if the fd was closed (or was already released), - * {@code false} if the OS reported a close failure and the - * flock may still be held + * @return {@code true} if the lock was explicitly released (or was already + * released), {@code false} if the OS reported an unlock failure */ - public boolean release() { + public synchronized boolean release() { if (fd < 0) { return true; } - if (Files.close(fd) == 0) { + if (release0(fd) == 0) { fd = -1; return true; } @@ -180,6 +181,8 @@ private static String readHolder(String pidPath) { } } + private static native int release0(int fd); + private static void writePid(String pidPath) { long pid; try { diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index f11a5912..71c773cb 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -148,6 +148,11 @@ public final class SenderPool implements AutoCloseable { private final Condition slotReleased; // True iff the configuration enables store-and-forward (sf_dir set). private final boolean storeAndForward; + // Test seam: runs immediately before a capacity-starved borrow enters its + // condition wait, while it still holds the pool lock. Null in production; + // concurrency tests use a latch here to prove that several borrowers have + // all reached the wait path before recovering retired capacity. + private volatile Runnable beforeBorrowWaitHook; // Test seam: runs after a capacity-starved borrow's condition wait has // exhausted its positive timeout, before the loop's terminal pass. Null in // production; regression tests release a retired slot here to prove that @@ -822,6 +827,10 @@ public PooledSender borrow() { "timed out waiting for a Sender from the pool after " + acquireTimeoutMillis + "ms"); } try { + Runnable beforeWaitHook = beforeBorrowWaitHook; + if (beforeWaitHook != null) { + beforeWaitHook.run(); + } remainingNanos = slotReleased.awaitNanos(remainingNanos); if (remainingNanos <= 0) { Runnable hook = borrowWaitExpiredHook; @@ -841,6 +850,11 @@ public PooledSender borrow() { } } + @TestOnly + public void setBeforeBorrowWaitHook(Runnable hook) { + this.beforeBorrowWaitHook = hook; + } + @TestOnly public void setBorrowWaitExpiredHook(Runnable hook) { this.borrowWaitExpiredHook = hook; @@ -1397,7 +1411,15 @@ private boolean reprobeRetiredSlots() { for (int i = retiredSlots.size() - 1; i >= 0; i--) { SenderSlot s = retiredSlots.get(i); if (flockReleased(s)) { - retiredSlots.remove(i); + // Order is irrelevant. Swap with the tail before removing so + // each recovery does O(1) list work instead of shifting the + // remaining retired entries. The tail has already been probed + // by this reverse scan and, if still present, is unreleased. + int last = retiredSlots.size() - 1; + if (i < last) { + retiredSlots.set(i, retiredSlots.get(last)); + } + retiredSlots.remove(last); leakedSlots--; freeSlotIndex(s.slotIndex()); recovered = true; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java index 9f1500f2..59d0f83d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java @@ -35,8 +35,10 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; /** * Regression tests for the close() drain semantics. @@ -77,6 +79,56 @@ public void testCloseBlocksUntilAckArrives() throws Exception { } } + @Test + public void testCloseStartedHookRunsAfterClosedStateTransition() throws Exception { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicReference closerFailure = new AtomicReference<>(); + AtomicReference hookFailure = new AtomicReference<>(); + sender.setCloseStartedHook(() -> { + try { + try { + sender.table("must_reject_after_close_started"); + Assert.fail("close-started hook ran before closed=true was published"); + } catch (LineSenderException expected) { + // Required lifecycle boundary: public operations already reject. + } + entered.countDown(); + if (!release.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("close-started hook release timed out"); + } + } catch (Throwable t) { + hookFailure.set(t); + entered.countDown(); + } + }); + + Assert.assertEquals("installing the hook must not emit a pre-invocation witness", + 1L, entered.getCount()); + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + closerFailure.set(t); + } + }, "close-started-hook-test"); + try { + closer.start(); + Assert.assertTrue("close() never reached its internal lifecycle witness", + entered.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("close() completed while its internal witness was held", + closer.isAlive()); + } finally { + release.countDown(); + closer.join(10_000L); + sender.close(); + } + Assert.assertFalse("close thread did not finish", closer.isAlive()); + Assert.assertNull("close-started hook failed", hookFailure.get()); + Assert.assertNull("close() failed", closerFailure.get()); + } + @Test public void testCloseFastWhenTimeoutIsZero() throws Exception { // Same delayed-ACK server, but with close_flush_timeout_millis=0 diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java new file mode 100644 index 00000000..5bfb0863 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java @@ -0,0 +1,141 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpWebSocketSenderCursorEngineAttachmentTest { + + private static final long SEGMENT_SIZE = 64 * 1024L; + + @Rule + public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testNullDoesNotDetachAnAttachedEngine() throws Exception { + assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + try { + sender.setCursorEngine(engine, true); + assertSecondAttachmentRejected(sender, null, false); + sender.close(); + Assert.assertTrue("sender must retain ownership after rejected detach", + engine.isCloseCompleted()); + } finally { + sender.close(); + engine.close(); + } + }); + } + + @Test + public void testOwnedEngineCannotBeReplacedAndItsSlotIsReleased() throws Exception { + assertMemoryLeak(() -> { + File slotDir = new File(sfDir.getRoot(), "owned-slot"); + CursorSendEngine first = null; + CursorSendEngine replacement = null; + CursorSendEngine reacquired = null; + QwpWebSocketSender sender = null; + boolean replacementRejected = false; + try { + first = new CursorSendEngine(slotDir.getAbsolutePath(), SEGMENT_SIZE); + replacement = new CursorSendEngine(null, SEGMENT_SIZE); + sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setCursorEngine(first, true); + try { + sender.setCursorEngine(replacement, true); + } catch (LineSenderException e) { + replacementRejected = true; + Assert.assertTrue(e.getMessage().contains("already attached")); + } + + sender.close(); + sender = null; + try { + reacquired = new CursorSendEngine(slotDir.getAbsolutePath(), SEGMENT_SIZE); + } catch (IllegalStateException e) { + throw new AssertionError("sender close did not release the first owned " + + "engine's slot after a second attachment attempt", e); + } + Assert.assertTrue("second attachment must be rejected", replacementRejected); + } finally { + if (sender != null) { + sender.close(); + } + if (reacquired != null) { + reacquired.close(); + } + if (replacement != null) { + replacement.close(); + } + if (first != null) { + first.close(); + } + } + }); + } + + @Test + public void testSameSharedEngineCannotTransferOwnership() throws Exception { + assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + try { + sender.setCursorEngine(engine, false); + assertSecondAttachmentRejected(sender, engine, true); + sender.close(); + Assert.assertFalse("rejected attachment must not transfer ownership", + engine.isCloseCompleted()); + } finally { + sender.close(); + engine.close(); + } + }); + } + + private static void assertSecondAttachmentRejected( + QwpWebSocketSender sender, + CursorSendEngine engine, + boolean takeOwnership + ) { + try { + sender.setCursorEngine(engine, takeOwnership); + Assert.fail("expected second cursor engine attachment to be rejected"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("already attached")); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index 0a6c0895..f2c560ea 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -195,12 +195,11 @@ public void testSlotLockNotReleasedWhenIoThreadRefusesToStop() throws Exception } /** - * Manager-worker leak path: engine close retains the slot while a shared + * Manager-worker handoff path: engine close retains the slot while a shared * manager is still inside a service pass. The sender must expose that * retained flock to SenderPool. Repeated sender close calls remain no-ops; - * the test cleans the deliberately retained engine up directly — and once - * that cleanup completes, the sender must expose the released flock so a - * pool that retired the slot can recover its capacity. + * the manager pass owns deferred cleanup and releases the flock when it + * finishes, so the sender can expose recovery without a direct close retry. */ @Test(timeout = 30_000L) public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception { @@ -211,6 +210,7 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception String slot = tmpDir + "/slot"; long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); CountDownLatch workerBlocked = new CountDownLatch(1); CountDownLatch releaseWorker = new CountDownLatch(1); AtomicBoolean fired = new AtomicBoolean(); @@ -219,6 +219,7 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception QwpWebSocketSender wss = null; boolean managerClosed = false; try { + manager.setAfterRingCleanupHook(cleanupFinished::countDown); manager.setBeforeInstallSyncHook(() -> { if (!fired.compareAndSet(false, true)) return; workerBlocked.countDown(); @@ -262,12 +263,9 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception engine.isCloseCompleted()); releaseWorker.countDown(); - manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); - // Test-only cleanup through the retained local reference (the - // shared-manager path has no worker-exit handoff to defer to; - // a retried close() is its reclaim driver). - engine.close(); - Assert.assertTrue("direct cleanup did not complete after worker exit", + Assert.assertTrue("shared-manager pass did not finish deferred cleanup", + cleanupFinished.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("deferred cleanup did not complete after the ring pass", engine.isCloseCompleted()); // Recovery contract: the sender re-probes its retained engine, // so the completed cleanup (and released flock) MUST become @@ -276,7 +274,7 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception Assert.assertTrue("sender must expose the late flock release to the pool", wss.isSlotLockReleased()); try (SlotLock probe = SlotLock.acquire(slot)) { - Assert.assertNotNull("slot must be acquirable after direct cleanup", probe); + Assert.assertNotNull("slot must be acquirable after deferred cleanup", probe); } manager.close(); @@ -285,6 +283,7 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception throw new AssertionError("install hook failed", hookErr.get()); } } finally { + manager.setAfterRingCleanupHook(null); manager.setBeforeInstallSyncHook(null); releaseWorker.countDown(); if (wss != null) { @@ -293,12 +292,6 @@ public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception } catch (Throwable ignored) { } } - if (engine != null && !engine.isCloseCompleted()) { - try { - engine.close(); - } catch (Throwable ignored) { - } - } if (!managerClosed) { manager.close(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 38cd1bcf..6c08aeff 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -88,7 +88,8 @@ public void tearDown() { * inside a service pass for the engine's ring, {@code close()} must NOT * hand the slot to anyone else. With the quiescence barrier reverted, * close() releases the slot lock immediately and the mid-test - * {@code SlotLock.acquire} probe succeeds — failing the test. + * {@code SlotLock.acquire} probe succeeds. Once the pass finishes, its + * deferred cleanup must release the slot without a direct close retry. */ @Test(timeout = 30_000L) public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { @@ -98,6 +99,7 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { // 60 s poll: the worker only acts when explicitly woken, so the // single pass we park below is the only pass in flight. SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); CountDownLatch workerBlocked = new CountDownLatch(1); CountDownLatch releaseWorker = new CountDownLatch(1); AtomicBoolean fired = new AtomicBoolean(); @@ -105,6 +107,7 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { boolean managerClosed = false; CursorSendEngine engine = null; try { + manager.setAfterRingCleanupHook(cleanupFinished::countDown); manager.setBeforeInstallSyncHook(() -> { if (!fired.compareAndSet(false, true)) return; workerBlocked.countDown(); @@ -149,19 +152,16 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { // Let the worker finish its pass (it abandons the spare: the // ring was deregistered by the close attempt above). releaseWorker.countDown(); - manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); - - // Retry close(): the barrier now succeeds and the full cleanup - // (ring, watermark, unlink, slot lock) must complete. - engine.close(); - Assert.assertTrue("retried close must report complete cleanup", + Assert.assertTrue("ring pass did not finish deferred cleanup", + cleanupFinished.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("ring-pass cleanup must report complete cleanup", engine.isCloseCompleted()); engine = null; try (SlotLock probe = SlotLock.acquire(slot)) { Assert.assertNotNull("slot must be acquirable after a completed close", probe); } catch (Exception e) { - throw new AssertionError("retried close() did not release the slot lock", e); + throw new AssertionError("ring-pass cleanup did not release the slot lock", e); } manager.close(); @@ -170,14 +170,9 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { throw new AssertionError("install hook failed", hookErr.get()); } } finally { + manager.setAfterRingCleanupHook(null); manager.setBeforeInstallSyncHook(null); releaseWorker.countDown(); - if (engine != null) { - try { - engine.close(); - } catch (Throwable ignored) { - } - } if (!managerClosed) { manager.close(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index f4de1ff2..c11fe1fe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -197,6 +197,25 @@ public void testCloseIsIdempotent() throws Exception { }); } + @Test + public void testCallbackCreationFailurePrecedesOwnedManagerResources() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine.setBeforeDeferredCloseCreationHook(() -> { + throw new OutOfMemoryError("simulated bound callback allocation failure"); + }); + try { + try { + new CursorSendEngine(tmpDir, 4096); + fail("expected callback allocation failure"); + } catch (OutOfMemoryError expected) { + assertEquals("simulated bound callback allocation failure", expected.getMessage()); + } + } finally { + CursorSendEngine.setBeforeDeferredCloseCreationHook(null); + } + }); + } + @Test public void testConstructorFailureAfterOwnedManagerStartCleansResources() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java index 5dcf6def..c142dd6a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -178,21 +178,18 @@ public void testCloseCompletedPublishedOnlyAfterConfirmedFlockRelease() throws E /** * The error half of the publish-after-release contract: when - * {@code SlotLock.release()} reports failure (the OS refused the fd - * close, so the flock may still be held), {@code closeCompleted} must - * NEVER be published — not by the failing close(), and not by a retried - * close() either (the terminal-cleanup claim is already consumed; the - * design deliberately leaves an unconfirmed release to the kernel's - * process-exit cleanup rather than risking a double release). A pool - * observing {@code isCloseCompleted() == false} keeps the slot retired, - * which is exactly right: the flock genuinely is still held. + * {@code SlotLock.release()} reports failure (the OS refused the explicit + * unlock, so the flock may still be held), {@code closeCompleted} must + * stay false until a later close confirms release. A pool observing + * {@code isCloseCompleted() == false} keeps the slot retired while the + * flock is genuinely held, then recovers it once the retry completes. *

    - * {@code Files.close} cannot be made to fail through the public API, so - * the lock's fd is swapped to a known-bad descriptor for the close and - * restored (and released for real) afterwards. + * The lock's fd is swapped to a known-bad descriptor for a deterministic + * native unlock failure, then restored before retrying through the engine's + * public close API. */ @Test(timeout = 30_000L) - public void testUnconfirmedFlockReleaseKeepsCloseIncomplete() throws Exception { + public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws Exception { TestUtils.assertMemoryLeak(() -> { CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock"); @@ -221,21 +218,24 @@ public void testUnconfirmedFlockReleaseKeepsCloseIncomplete() throws Exception { } catch (IllegalStateException expected) { // good — incomplete close really means "still locked". } - // A retried close() must neither throw nor re-run the terminal - // cleanup (the claim is consumed) nor suddenly report success. + + // Remove the injected fault. The retry must only re-attempt + // the retained fd release: ring/watermark cleanup stays + // exactly-once, while completion and slot reusability recover. + fdField.setInt(slotLock, realFd); engine.close(); - assertFalse("retried close() must not fabricate completion after a failed " - + "flock release — the claim is consumed and the kernel owns " - + "the eventual cleanup", + assertTrue("retried close() must complete after the flock release succeeds", engine.isCloseCompleted()); + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — eventual completion implies reusable capacity. + } } finally { - // Undo the fault and drop the real flock so the test leaves no - // open fd behind (in production the kernel does this at exit). - fdField.setInt(slotLock, realFd); - assertTrue("restored fd must release cleanly", slotLock.release()); - } - try (SlotLock ignored = SlotLock.acquire(sfDir)) { - // good — with the flock genuinely dropped, the slot is reusable. + // If an assertion failed before the successful retry, restore + // and release the real fd so the test never leaks a flock. + if (!engine.isCloseCompleted()) { + fdField.setInt(slotLock, realFd); + assertTrue("restored fd must release cleanly", slotLock.release()); + } } }); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java new file mode 100644 index 00000000..1bb6b899 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java @@ -0,0 +1,131 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +/** + * Standalone comparison of the service-pass ownership barriers used by + * SegmentManager before and after its per-entry state change. This deliberately + * has no pass/fail threshold: it reports target-JVM costs while the production + * state machine and lifecycle tests establish correctness. + * + * 

    + * mvn -pl core test-compile
+ * mvn -pl core exec:java -Dexec.classpathScope=test \
+ *   -Dexec.mainClass=io.questdb.client.test.cutlass.qwp.client.sf.cursor.SegmentManagerPassBarrierBenchmark \
+ *   -Dexec.args="--passes=10000000"
+ *
    + */ +public final class SegmentManagerPassBarrierBenchmark { + + private static final Object MONITOR = new Object(); + private static final AtomicIntegerFieldUpdater STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(Entry.class, "state"); + private static volatile long checksum; + private static volatile int serviceProbe; + + public static void main(String[] args) { + int passes = 10_000_000; + for (String arg : args) { + if (arg.startsWith("--passes=")) { + passes = Integer.parseInt(arg.substring("--passes=".length())); + } else { + throw new IllegalArgumentException("unknown argument: " + arg); + } + } + if (passes < 1) { + throw new IllegalArgumentException("passes must be positive"); + } + + int warmup = Math.max(100_000, passes / 10); + for (int rings : new int[]{1, 32, 256}) { + measureMonitor(rings, warmup); + measureAtomic(rings, warmup); + long monitorNanos = measureMonitor(rings, passes); + long atomicNanos = measureAtomic(rings, passes); + System.out.printf( + "rings=%d passes=%d monitor(two enters)=%.2f ns/pass atomic(two CAS)=%.2f ns/pass ratio=%.2f%n", + rings, + passes, + (double) monitorNanos / passes, + (double) atomicNanos / passes, + (double) monitorNanos / atomicNanos); + } + System.out.println("checksum=" + checksum); + } + + private static Entry[] entries(int count) { + Entry[] entries = new Entry[count]; + for (int i = 0; i < count; i++) { + entries[i] = new Entry(); + } + return entries; + } + + private static long measureAtomic(int rings, int passes) { + Entry[] entries = entries(rings); + long start = System.nanoTime(); + for (int i = 0; i < passes; i++) { + Entry entry = entries[i % rings]; + if (!STATE_UPDATER.compareAndSet(entry, 0, 1)) { + throw new AssertionError("claim failed"); + } + service(entry); + if (!STATE_UPDATER.compareAndSet(entry, 1, 0)) { + throw new AssertionError("completion failed"); + } + } + checksum = checksum * 31 + entries[passes % rings].state; + return System.nanoTime() - start; + } + + private static long measureMonitor(int rings, int passes) { + Entry[] entries = entries(rings); + long start = System.nanoTime(); + for (int i = 0; i < passes; i++) { + Entry entry = entries[i % rings]; + synchronized (MONITOR) { + entry.state = 1; + } + service(entry); + synchronized (MONITOR) { + entry.state = 0; + } + } + checksum = checksum * 31 + entries[passes % rings].state; + return System.nanoTime() - start; + } + + private static void service(Entry entry) { + // Models the non-trivial serviceRing0 call between claim and complete + // and prevents the JVM from coarsening both monitor regions into one. + serviceProbe = entry.state; + } + + private static final class Entry { + volatile int state; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 015cb0d8..20930f50 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -124,18 +124,16 @@ public void testReleaseConfirmsAndIsIdempotent() throws Exception { } /** - * The {@code release() == false} branch: when the OS reports a close - * failure, release must (a) return {@code false} so owners gating a - * "slot reusable" signal never see a lie, (b) RETAIN the fd — forgetting - * it would misreport the lock state and forfeit any later retry — and - * (c) keep returning {@code false} on repeat attempts while the failure - * persists. Once the close succeeds, release confirms and stays - * confirmed. {@code Files.close} cannot be made to fail through the - * public API, so the fd is swapped to a known-bad descriptor and - * restored afterwards. + * The {@code release() == false} branch: when the OS reports an explicit + * unlock failure, release must (a) return {@code false} so owners gating a + * "slot reusable" signal never see a lie, (b) retain the fd because the + * non-consuming unlock can safely be retried, and (c) keep returning + * {@code false} while the failure persists. Once unlock succeeds, release + * confirms and stays confirmed. Swapping in a known-bad descriptor gives + * the slot-specific native primitive a deterministic unlock failure. */ @Test - public void testFailedCloseRetainsFdAndReportsFalse() throws Exception { + public void testFailedUnlockRetainsFdAndReportsFalse() throws Exception { TestUtils.assertMemoryLeak(() -> { String slot = parentDir + "/failed-release"; SlotLock lock = SlotLock.acquire(slot); @@ -144,26 +142,26 @@ public void testFailedCloseRetainsFdAndReportsFalse() throws Exception { int realFd = fdField.getInt(lock); assertTrue("precondition: acquire must hold a live fd", realFd >= 0); try { - // A non-negative fd no process has open: close(2) fails EBADF. + // A non-negative descriptor no process has open makes the + // explicit flock/UnlockFileEx operation fail without consuming + // the real descriptor that continues to hold the slot lock. fdField.setInt(lock, 1_000_000_000); - assertFalse("release must report false when the OS close fails", + assertFalse("release must report false when explicit unlock fails", lock.release()); - assertEquals("failed release must retain the fd for a retry — " - + "dropping it would misreport the flock as released", + assertEquals("failed unlock must retain the fd for a safe retry", 1_000_000_000, fdField.getInt(lock)); - assertFalse("repeat release must keep reporting false while the failure persists", + assertFalse("repeat release must stay false while unlock keeps failing", lock.release()); - // While the release is unconfirmed the REAL flock is still - // held — a second acquire on the slot must fail. + // While the release is unconfirmed the real flock remains held. try (SlotLock ignored = SlotLock.acquire(slot)) { fail("slot must not be acquirable while the original flock fd is still open"); } catch (IllegalStateException expected) { - // good — unconfirmed release really means "still locked". + // good - unconfirmed release really means "still locked". } } finally { fdField.setInt(lock, realFd); } - assertTrue("release must confirm once the close succeeds", lock.release()); + assertTrue("release must confirm once explicit unlock succeeds", lock.release()); assertTrue("confirmed release must stay confirmed", lock.release()); try (SlotLock again = SlotLock.acquire(slot)) { assertEquals(slot, again.slotDir()); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index a268b14a..6db9e009 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -30,9 +30,11 @@ import ch.qos.logback.core.read.ListAppender; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import io.questdb.client.std.Files; @@ -50,6 +52,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Paths; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -774,6 +777,285 @@ public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { }); } + @Test + public void testMixedRetiredSlotsRecoverWithoutLosingAccounting() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, 8, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[8]; + Sender[] delegates = new Sender[8]; + for (int i = 0; i < leases.length; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + } + for (int i = 0; i < leases.length; i++) { + // Release the real native resources, then forge the + // delayed publication which makes the pool retire the + // slot. The idempotent second close in discardBroken() + // leaves the forged state unchanged. + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals(8, pool.leakedSlotCount()); + + // Mix completed and incomplete entries throughout the + // retired list. A recovery pass must remove exactly these + // four without skipping a swapped entry or corrupting the + // bitmap/count relationship. + int[] released = {0, 2, 5, 7}; + for (int i = 0; i < released.length; i++) { + setBooleanField(delegates[released[i]], "slotLockReleased", true); + } + pool.reapIdle(); + + Assert.assertEquals(4, pool.leakedSlotCount()); + Assert.assertEquals(4, ((List) getField(pool, "retiredSlots")).size()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + for (int i = 0; i < slotInUse.length; i++) { + boolean mustRemainRetired = i == 1 || i == 3 || i == 4 || i == 6; + Assert.assertEquals("slot reservation mismatch at index " + i, + mustRemainRetired, slotInUse[i]); + } + + // Exactly the four restored reservations are reusable. + PooledSender[] recovered = new PooledSender[4]; + try { + for (int i = 0; i < recovered.length; i++) { + recovered[i] = pool.borrow(); + } + Assert.assertEquals(4, pool.totalSize()); + Assert.assertEquals(4, pool.leakedSlotCount()); + } finally { + for (int i = 0; i < recovered.length; i++) { + if (recovered[i] != null) { + recovered[i].close(); + } + } + } + } + } + }); + } + + @Test(timeout = 30_000) + public void testMultipleWaitingBorrowersWakeForStaggeredRetiredSlotRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, 6, 10_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[6]; + Sender[] delegates = new Sender[6]; + for (int i = 0; i < leases.length; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + } + for (int i = 0; i < leases.length; i++) { + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals("all capacity must start retired", + 6, pool.leakedSlotCount()); + + final int borrowerCount = 3; + CountDownLatch allAcquired = new CountDownLatch(borrowerCount); + CountDownLatch allDone = new CountDownLatch(borrowerCount); + CountDownLatch firstAcquired = new CountDownLatch(1); + CountDownLatch initialWaiters = new CountDownLatch(borrowerCount); + CountDownLatch releaseBorrowers = new CountDownLatch(1); + CountDownLatch reparkedWaiters = new CountDownLatch(borrowerCount - 1); + AtomicReference failure = new AtomicReference<>(); + ConcurrentHashMap initialWaiterThreads = new ConcurrentHashMap<>(); + ConcurrentHashMap reparkedWaiterThreads = new ConcurrentHashMap<>(); + PooledSender[] recovered = new PooledSender[borrowerCount]; + pool.setBeforeBorrowWaitHook(() -> { + if (initialWaiterThreads.putIfAbsent(Thread.currentThread(), Boolean.TRUE) == null) { + initialWaiters.countDown(); + } + }); + for (int i = 0; i < borrowerCount; i++) { + final int borrower = i; + Thread thread = new Thread(() -> { + try { + recovered[borrower] = pool.borrow(); + allAcquired.countDown(); + firstAcquired.countDown(); + if (!releaseBorrowers.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to release recovered borrower"); + } + } catch (Throwable e) { + failure.compareAndSet(null, e); + } finally { + try { + if (recovered[borrower] != null) { + recovered[borrower].close(); + } + } catch (Throwable e) { + failure.compareAndSet(null, e); + } finally { + allDone.countDown(); + } + } + }, "sender-pool-retired-waiter-" + i); + thread.start(); + } + + try { + Assert.assertTrue("all borrowers must reach the condition wait", + initialWaiters.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("three distinct borrowers must reach the wait path", + borrowerCount, initialWaiterThreads.size()); + + // The hook runs while each borrower holds the pool lock, + // immediately before awaitNanos atomically releases that + // lock and enqueues it. Once the last latch count lands, + // the first reapIdle() cannot acquire the lock until all + // three borrowers are definitely condition waiters. + pool.setBeforeBorrowWaitHook(() -> { + if (reparkedWaiterThreads.putIfAbsent(Thread.currentThread(), Boolean.TRUE) == null) { + reparkedWaiters.countDown(); + } + }); + setBooleanField(delegates[2], "slotLockReleased", true); + pool.reapIdle(); + + // One restored index admits exactly one borrower. The + // other two must consume signalAll(), lose the capacity + // race, and deterministically re-enter the wait path. + Assert.assertTrue("one borrower must take the first restored index", + firstAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("two distinct borrowers must re-park after the first recovery", + reparkedWaiters.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("exactly two distinct borrowers must re-enter the wait path", + borrowerCount - 1, reparkedWaiterThreads.size()); + Assert.assertEquals("exactly one borrower should hold restored capacity", + borrowerCount - 1, allAcquired.getCount()); + + // Recover two non-contiguous entries in one reverse / + // swap-remove pass. A single signal() here would wake + // only one of the two proven waiters; signalAll() must + // wake both and let them claim the two restored indices. + pool.setBeforeBorrowWaitHook(null); + setBooleanField(delegates[0], "slotLockReleased", true); + setBooleanField(delegates[5], "slotLockReleased", true); + pool.reapIdle(); + Assert.assertTrue("all waiting borrowers must receive restored capacity", + allAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertEquals(3, pool.totalSize()); + Assert.assertEquals(3, pool.leakedSlotCount()); + + boolean[] seen = new boolean[6]; + for (int i = 0; i < recovered.length; i++) { + int slotIndex = getIntField(slotOf(recovered[i]), "slotIndex"); + Assert.assertFalse("borrowers must receive distinct restored indices", + seen[slotIndex]); + seen[slotIndex] = true; + } + Assert.assertTrue("slot 0 must be restored", seen[0]); + Assert.assertTrue("slot 2 must be restored", seen[2]); + Assert.assertTrue("slot 5 must be restored", seen[5]); + if (failure.get() != null) { + throw new AssertionError("waiting borrower failed", failure.get()); + } + } finally { + pool.setBeforeBorrowWaitHook(null); + releaseBorrowers.countDown(); + Assert.assertTrue("borrower threads must finish", + allDone.await(10, TimeUnit.SECONDS)); + } + if (failure.get() != null) { + throw new AssertionError("waiting borrower failed", failure.get()); + } + } + } + }); + } + + @Test + public void testPoolRetiresAndRecoversSlotThroughFailedFlockReleaseRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 1, 2_000, + Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Sender delegate = getDelegate(a); + CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); + SlotLock slotLock = (SlotLock) getField(engine, "slotLock"); + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(slotLock); + Assert.assertTrue("precondition: live flock fd", realFd >= 0); + try { + // Inject one persistent explicit-unlock failure. + // Delegate close must retire the only pool slot rather + // than publish a release while the real flock remains held. + synchronized (slotLock) { + fdField.setInt(slotLock, 1_000_000_000); + } + invokeDiscardBroken(pool, a); + Assert.assertEquals("failed release must retire pool capacity", + 1, pool.leakedSlotCount()); + Assert.assertFalse(engine.isCloseCompleted()); + + // Remove the fault without calling close again: the + // engine's error-path retry driver runs outside the + // pool lock and must eventually publish completion. + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + } + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("flock-release retry never completed"); + } + Thread.sleep(1L); + } + + // A capacity-starved public borrow re-probes the + // retained delegate, frees index 0, and proves the + // recovered flock is genuinely acquirable. + PooledSender recovered = pool.borrow(); + try { + Assert.assertEquals("retry must restore pool capacity", + 0, pool.leakedSlotCount()); + Assert.assertEquals(1, countSlotDirs()); + } finally { + recovered.close(); + } + } finally { + if (!engine.isCloseCompleted()) { + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + Assert.assertTrue("restored fd must release cleanly", + slotLock.release()); + } + } + } + } + } + }); + } + @Test public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws Exception { // Full-stack twin of the two forged-flag recovery tests above: no @@ -830,6 +1112,13 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws // delegate's engine close times out its bounded join, // hands cleanup to the worker's exit path, and reports // the retained flock; the pool must retire the slot. + // Fault the old callback-allocation/registration path. + // C3 makes the owned-engine handoff preallocated, so + // this hook must no longer be reached by production + // teardown. + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated callback allocation failure"); + }); manager.setWorkerJoinTimeoutMillis(50L); invokeDiscardBroken(pool, a); Assert.assertEquals( @@ -839,18 +1128,15 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws Assert.assertFalse("engine cleanup must still be pending", engine.isCloseCompleted()); - // Un-wedge: the worker finishes its pass, exits (its - // loop was stopped by the close), and runs the - // deferred cleanup that releases the real flock. + // Un-wedge and deterministically reap the manager. + // No sender/engine close retry is allowed: worker exit + // itself must own and finish the preallocated handoff. releaseWorker.countDown(); - long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); - while (!engine.isCloseCompleted()) { - if (System.nanoTime() > deadlineNs) { - throw new AssertionError( - "deferred engine cleanup never ran on manager-worker exit"); - } - Thread.sleep(1); - } + manager.close(); + Assert.assertTrue("manager worker must be reaped", manager.isWorkerReaped()); + Assert.assertTrue("worker exit must run deferred cleanup despite the " + + "old allocation fault injection", + engine.isCloseCompleted()); // The housekeeper tick is the recovery driver. pool.reapIdle(); @@ -876,6 +1162,7 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws throw new AssertionError("trim hook failed", hookErr.get()); } } finally { + manager.setBeforeExitCleanupRegistrationHook(null); manager.setBeforeTrimSyncHook(null); releaseWorker.countDown(); } @@ -884,6 +1171,16 @@ public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws }); } + @Test + public void testPreallocatedExitHandoffCleansInRangeStartupRecoverer() throws Exception { + assertPreallocatedExitHandoffCleansStartupRecoverer(0, 1); + } + + @Test + public void testPreallocatedExitHandoffCleansOutOfRangeStartupRecoverer() throws Exception { + assertPreallocatedExitHandoffCleansStartupRecoverer(1, 1); + } + // ---------------------------------------------------------------------- // Recovery: stable slot ids let a re-created pool re-adopt unacked data. // ---------------------------------------------------------------------- @@ -983,6 +1280,92 @@ public void testRecoveryDelegateForcesOffInitialConnectMode() throws Exception { }); } + @Test + public void testSharedManagerPassCompletionRecoversRetiredPoolSlot() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + long segSize = io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment.HEADER_SIZE + + io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment.FRAME_HEADER_SIZE + 32L; + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + AtomicReference engineRef = new AtomicReference<>(); + manager.setAfterRingCleanupHook(cleanupFinished::countDown); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) { + return; + } + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertEquals(0, Files.mkdir(sfDir, Files.DIR_MODE_DEFAULT)); + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + IntFunction factory = slotIndex -> { + CursorSendEngine engine = new CursorSendEngine( + slot("default-" + slotIndex), segSize, manager); + engineRef.set(engine); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", port); + sender.setCursorEngine(engine, true); + return sender; + }; + SenderPool pool = new SenderPool( + config, 0, 1, 500, 0, Long.MAX_VALUE, factory); + try { + PooledSender lease = pool.borrow(); + Assert.assertTrue("shared manager never entered the sender ring's service pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + lease.close(); + + manager.setWorkerJoinTimeoutMillis(50L); + pool.reapIdle(); + Assert.assertEquals("pool must retire the slot while its shared-manager pass is live", + 1, pool.leakedSlotCount()); + CursorSendEngine engine = engineRef.get(); + Assert.assertFalse("engine cleanup must remain pending during the live pass", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + Assert.assertTrue("deferred cleanup did not finish with the ring pass", + cleanupFinished.await(5, TimeUnit.SECONDS)); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + + pool.reapIdle(); + Assert.assertEquals("pass completion must drive deferred engine cleanup and restore capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("deferred cleanup must publish the released flock", + engine.isCloseCompleted()); + try (PooledSender recovered = pool.borrow()) { + Assert.assertTrue("recovered slot must be reusable", Files.exists(slot("default-0"))); + } + } finally { + releaseWorker.countDown(); + manager.setAfterRingCleanupHook(null); + manager.setBeforeInstallSyncHook(null); + pool.close(); + manager.close(); + } + } + }); + } + @Test public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() throws Exception { // C1 regression: the startup recovery loop MUST mirror discardBroken / @@ -2318,6 +2701,134 @@ private String slot(String name) { return sfDir + "/" + name; } + private void assertPreallocatedExitHandoffCleansStartupRecoverer( + int strandedIndex, int maxSize) throws Exception { + TestUtils.assertMemoryLeak(() -> { + String strandedId = "default-" + strandedIndex; + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=100;"; + Sender sender = Sender.builder(config).senderId(strandedId).build(); + sender.table("recover").longColumn("v", strandedIndex).atNow(); + sender.flush(); + try { + sender.close(); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains("drain timed out")); + } + } + Assert.assertTrue("startup-recovery fixture must contain an unacked segment", + hasSegmentFile(slot(strandedId))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicBoolean instrumented = new AtomicBoolean(); + AtomicReference engineRef = new AtomicReference<>(); + AtomicReference managerRef = new AtomicReference<>(); + AtomicReference hookErr = new AtomicReference<>(); + CountDownLatch releaseWorker = new CountDownLatch(1); + IntFunction factory = idx -> { + Sender sender = Sender.builder(config).senderId("default-" + idx).build(); + if (idx == strandedIndex && instrumented.compareAndSet(false, true)) { + try { + CursorSendEngine engine = (CursorSendEngine) getField(sender, "cursorEngine"); + SegmentManager manager = (SegmentManager) getField(engine, "manager"); + CountDownLatch workerBlocked = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + manager.setBeforeTrimSyncHook(() -> { + if (!fired.compareAndSet(false, true)) { + return; + } + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting to release recovery worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + if (!workerBlocked.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("recovery manager never entered a service pass"); + } + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated callback allocation failure"); + }); + manager.setWorkerJoinTimeoutMillis(50L); + engineRef.set(engine); + managerRef.set(manager); + } catch (Throwable t) { + try { + sender.close(); + } catch (Throwable ignored) { + } + throw new RuntimeException(t); + } + } + return sender; + }; + + SenderPool pool = null; + try { + pool = newPoolWithFactory(config, 0, maxSize, 2_000, factory); + CursorSendEngine engine = engineRef.get(); + SegmentManager manager = managerRef.get(); + Assert.assertNotNull("startup recovery must build the stranded slot", engine); + if (strandedIndex < maxSize) { + Assert.assertEquals("in-range recoverer must remain retired while worker is live", + 1, pool.leakedSlotCount()); + } else { + Assert.assertEquals("out-of-range recovery must not consume pool capacity", + 0, pool.leakedSlotCount()); + } + Assert.assertFalse("cleanup must remain pending while the worker is live", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + manager.close(); + Assert.assertTrue("recovery manager worker must be reaped", manager.isWorkerReaped()); + Assert.assertTrue("worker exit must complete startup-recoverer cleanup without " + + "a sender or engine close retry [index=" + strandedIndex + "]", + engine.isCloseCompleted()); + if (hookErr.get() != null) { + throw new AssertionError("recovery worker hook failed", hookErr.get()); + } + if (strandedIndex < maxSize) { + pool.reapIdle(); + Assert.assertEquals("late cleanup must restore in-range pool capacity", + 0, pool.leakedSlotCount()); + } + try (SlotLock ignored = SlotLock.acquire(slot(strandedId))) { + // Completion must mean the real slot flock is reusable. + } + } finally { + releaseWorker.countDown(); + SegmentManager manager = managerRef.get(); + if (manager != null) { + manager.setBeforeExitCleanupRegistrationHook(null); + manager.setBeforeTrimSyncHook(null); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + manager.close(); + } + CursorSendEngine engine = engineRef.get(); + if (engine != null && !engine.isCloseCompleted()) { + engine.close(); + } + if (pool != null) { + pool.close(); + } + } + } + }); + } + private int countSlotDirs() { if (!Files.exists(sfDir)) { return 0; From a0865758215b20aecceb963ae78d2f3826add3a8 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 11 Jul 2026 21:29:48 +0100 Subject: [PATCH 15/26] fix(qwp): harden cleanup and recovery lifecycle --- core/src/main/c/share/net.c | 5 + core/src/main/c/share/net.h | 8 + core/src/main/c/windows/net.c | 9 + .../cutlass/http/client/WebSocketClient.java | 9 + .../qwp/client/QwpWebSocketSender.java | 48 ++ .../client/sf/cursor/CursorSendEngine.java | 145 +++++- .../sf/cursor/CursorWebSocketSendLoop.java | 26 +- .../qwp/client/sf/cursor/SegmentManager.java | 107 ++-- .../qwp/client/sf/cursor/SegmentRing.java | 31 ++ .../io/questdb/client/impl/SenderPool.java | 71 ++- .../client/network/JavaTlsClientSocket.java | 8 + .../java/io/questdb/client/network/Net.java | 2 + .../questdb/client/network/NetworkFacade.java | 9 + .../client/network/NetworkFacadeImpl.java | 5 + .../questdb/client/network/PlainSocket.java | 13 +- .../io/questdb/client/network/Socket.java | 11 + .../cutlass/qwp/client/CloseDrainTest.java | 77 +++ ...WebSocketSendLoopBlockedSendCloseTest.java | 256 ++++++++++ .../cursor/FlockReleaseRetryDriverTest.java | 247 ++++++++++ .../SegmentManagerUnlinkFailureTest.java | 350 +++++++++++++ .../qwp/websocket/TestWebSocketServer.java | 15 + .../client/test/impl/SenderPoolSfTest.java | 172 +++++++ .../network/SocketTrafficShutdownTest.java | 463 ++++++++++++++++++ 23 files changed, 1987 insertions(+), 100 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java create mode 100644 core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java diff --git a/core/src/main/c/share/net.c b/core/src/main/c/share/net.c index 3b0162fc..9b58fd65 100644 --- a/core/src/main/c/share/net.c +++ b/core/src/main/c/share/net.c @@ -128,6 +128,11 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_send return com_questdb_network_Net_EOTHERDISCONNECT; } +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *e, jclass cl, jint fd) { + return shutdown((int) fd, SHUT_RDWR); +} + JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv (JNIEnv *e, jclass cl, jint fd, jlong ptr, jint len) { ssize_t n; diff --git a/core/src/main/c/share/net.h b/core/src/main/c/share/net.h index 27143639..d80617e2 100644 --- a/core/src/main/c/share/net.h +++ b/core/src/main/c/share/net.h @@ -80,6 +80,14 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_send (JNIEnv *, jclass, jint, jlong, jint); +/* + * Class: io_questdb_client_network_Net + * Method: shutdown + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *, jclass, jint); + /* * Class: io_questdb_client_network_Net * Method: getSndBuf diff --git a/core/src/main/c/windows/net.c b/core/src/main/c/windows/net.c index fd290629..9cc98000 100644 --- a/core/src/main/c/windows/net.c +++ b/core/src/main/c/windows/net.c @@ -230,6 +230,15 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_configureNonBlocking return res; } +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *e, jclass cl, jint fd) { + const int result = shutdown((SOCKET) fd, SD_BOTH); + if (result == SOCKET_ERROR) { + SaveLastError(); + } + return result; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv (JNIEnv *e, jclass cl, jint fd, jlong addr, jint len) { const int n = recv((SOCKET) fd, (char *) addr, len, 0); diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index fd16dca7..94018fff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -248,6 +248,15 @@ public void close() { } } + /** + * Shuts down socket traffic without releasing the descriptor or freeing + * buffers that a concurrent I/O worker may still access. The owner must + * call {@link #close()} after joining the worker to complete cleanup. + */ + public void closeTraffic() { + socket.closeTraffic(); + } + /** * Connects to a WebSocket server. * diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 3586a4d2..9f454c9c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -189,6 +189,9 @@ public class QwpWebSocketSender implements Sender { // Null in production; set reflectively by tests. @TestOnly private volatile java.util.function.Supplier clientFactoryOverride; + // Test-only lifecycle witness. drainOnClose() invokes and clears it after + // confirming a real unacknowledged close target, immediately before waiting. + private volatile Runnable closeDrainWaitingHook; // close() drain timeout in millis. Default applied at construction. // 0 or -1 means "fast close" (skip the drain); otherwise close blocks // up to this many millis for ackedFsn to catch up to publishedFsn. @@ -290,6 +293,9 @@ public class QwpWebSocketSender implements Sender { // re-probe retired slots to recover their capacity. volatile: written on // the closing thread, read by pool threads. private volatile boolean slotLockReleased; + // Optional owning-pool notification, relayed after the engine confirms the + // flock release and slotLockReleased is visible to pool re-probes. + private volatile Runnable slotLockReleaseListener; // Engine whose close() could not complete during sender close() — its // cleanup is pending on a worker/I/O-thread exit path. isSlotLockReleased() // re-probes it so a late flock release becomes visible to the owning pool. @@ -1368,6 +1374,17 @@ public boolean isSlotLockReleased() { return false; } + /** + * Registers a callback for confirmed SF slot-lock release. Pools use this + * to wake borrowers without waiting for a timeout or housekeeper tick. + */ + public void setSlotLockReleaseListener(Runnable listener) { + slotLockReleaseListener = listener; + if (listener != null && slotLockReleased) { + listener.run(); + } + } + @Override public Sender decimalColumn(CharSequence name, Decimal64 value) { checkNotClosed(); @@ -2207,6 +2224,16 @@ public void setClientFactoryOverride(java.util.function.Supplier= target) { return; } + Runnable hook = closeDrainWaitingHook; + closeDrainWaitingHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable t) { + // A test witness must never prevent production resource cleanup. + LOG.error("Error in close-drain-waiting test hook: {}", String.valueOf(t)); + } + } long deadlineNanos = System.nanoTime() + closeFlushTimeoutMillis * 1_000_000L; while (cursorEngine.ackedFsn() < target) { // Stop on a latched terminal (acks will never reach target); @@ -3632,6 +3672,14 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm resetTableBuffersAfterFlush(keys); } + private void onSlotLockReleased() { + slotLockReleased = true; + Runnable listener = slotLockReleaseListener; + if (listener != null) { + listener.run(); + } + } + private void resetTableBuffersAfterFlush(ObjList keys) { for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 25699dcd..95addd39 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -30,6 +30,8 @@ import io.questdb.client.std.QuietCloseable; import org.jetbrains.annotations.TestOnly; +import java.util.ArrayDeque; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; @@ -65,7 +67,15 @@ public final class CursorSendEngine implements QuietCloseable { public static final long DEFAULT_APPEND_DEADLINE_NANOS = 30_000_000_000L; private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(CursorSendEngine.class); + private static final ThreadFactory DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY = + runnable -> new Thread(runnable, "qdb-sf-flock-release-retry"); + private static final Object FLOCK_RELEASE_RETRY_LOCK = new Object(); + private static final ArrayDeque FLOCK_RELEASE_RETRY_QUEUE = new ArrayDeque<>(); + private static volatile Runnable afterFlockReleaseRetryFailureHook; private static volatile Runnable beforeDeferredCloseCreationHook; + private static Thread flockReleaseRetryThread; + private static volatile ThreadFactory flockReleaseRetryThreadFactory = + DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY; private final long appendDeadlineNanos; // Number of times appendBlocking observed BACKPRESSURE_NO_SPARE on its first // ring.appendOrFsn attempt. One increment per blocking-call that had to wait @@ -136,6 +146,9 @@ public final class CursorSendEngine implements QuietCloseable { // isCloseCompleted() from pool threads re-probing a retired slot (see the // getter for why it must not synchronize). private volatile boolean closeCompleted; + // Invoked after closeCompleted publishes a confirmed flock release. Used + // by an owning sender pool to wake capacity-starved borrowers immediately. + private volatile Runnable slotLockReleaseListener; // Test-only hook run by finishClose() between the terminal cleanup and // the flock release. Lets a test park the releasing thread inside the // cleanup/release window and assert that closeCompleted stays false — @@ -143,8 +156,8 @@ public final class CursorSendEngine implements QuietCloseable { // held. volatile: finishClose may run on the manager worker's exit // thread while the hook is installed from a test thread. private volatile Runnable beforeFlockReleaseHook; - // Exactly one daemon drives a failed flock release to completion. The - // error path only: ordinary closes never allocate or start a thread. + // Ensures this engine has at most one entry in the shared flock-release + // retry driver. The error path only: ordinary closes never enqueue work. private final AtomicBoolean flockReleaseRetryStarted = new AtomicBoolean(); // Published before deferredClose is registered. The manager lock provides // the callback handoff fence; volatile also covers a direct test/retry read. @@ -781,6 +794,15 @@ private void finishClose(boolean fullyDrained) { } } + /** + * Installs a hook invoked after each failed shared-driver flock release. + * Test-only: makes persistent retry progress observable without sleeps. + */ + @TestOnly + public static void setAfterFlockReleaseRetryFailureHook(Runnable hook) { + afterFlockReleaseRetryFailureHook = hook; + } + /** * Installs a constructor fault hook immediately before the bound deferred * close callback is created. Test-only: proves callback allocation failure @@ -791,6 +813,23 @@ public static void setBeforeDeferredCloseCreationHook(Runnable hook) { beforeDeferredCloseCreationHook = hook; } + /** + * Overrides creation of the single shared flock-release retry thread. + * Test-only: makes thread creation/start failure and persistent retry + * scalability deterministic without relying on scheduler timing. + */ + @TestOnly + public static void setFlockReleaseRetryThreadFactory(ThreadFactory factory) { + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + if (flockReleaseRetryThread != null || !FLOCK_RELEASE_RETRY_QUEUE.isEmpty()) { + throw new IllegalStateException("flock-release retry driver is active"); + } + flockReleaseRetryThreadFactory = factory == null + ? DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY + : factory; + } + } + /** * Installs a hook that {@link #finishClose} runs between the terminal * cleanup and the slot-flock release. Test-only: makes the otherwise @@ -849,42 +888,97 @@ private boolean retryFlockReleaseIfReady() { } if (released) { closeCompleted = true; + Runnable listener = slotLockReleaseListener; + if (listener != null) { + try { + listener.run(); + } catch (Throwable ignored) { + // A notification failure must not invalidate a confirmed release. + } + } return true; } startFlockReleaseRetry(); return false; } + private static void runFlockReleaseRetryDriver() { + while (true) { + final int roundSize; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + roundSize = FLOCK_RELEASE_RETRY_QUEUE.size(); + if (roundSize == 0) { + flockReleaseRetryThread = null; + return; + } + } + boolean hasFailures = false; + for (int i = 0; i < roundSize; i++) { + final CursorSendEngine engine; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + engine = FLOCK_RELEASE_RETRY_QUEUE.pollFirst(); + } + if (engine.retryFlockReleaseIfReady()) { + engine.flockReleaseRetryStarted.set(false); + } else { + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + FLOCK_RELEASE_RETRY_QUEUE.addLast(engine); + } + hasFailures = true; + Runnable hook = afterFlockReleaseRetryFailureHook; + if (hook != null) { + hook.run(); + } + } + } + if (hasFailures) { + // Interruption must not abandon a retained flock, but clear + // the flag so subsequent parks still throttle retries. + Thread.interrupted(); + LockSupport.parkNanos(100_000_000L); + } + } + } + private void startFlockReleaseRetry() { if (!flockReleaseRetryStarted.compareAndSet(false, true)) { return; } - try { - Thread retryThread = new Thread(() -> { - while (!retryFlockReleaseIfReady()) { - try { - Thread.sleep(100L); - } catch (InterruptedException ignored) { - // A failed flock release must remain retryable until - // confirmed or process exit; interruption is not a - // safe reason to abandon the slot permanently. + Throwable startFailure = null; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + FLOCK_RELEASE_RETRY_QUEUE.addLast(this); + if (flockReleaseRetryThread == null) { + try { + Thread retryThread = flockReleaseRetryThreadFactory.newThread( + CursorSendEngine::runFlockReleaseRetryDriver); + if (retryThread == null) { + throw new IllegalStateException("retry thread factory returned null"); + } + retryThread.setDaemon(true); + flockReleaseRetryThread = retryThread; + retryThread.start(); + } catch (Throwable t) { + startFailure = t; + flockReleaseRetryThread = null; + CursorSendEngine queued; + while ((queued = FLOCK_RELEASE_RETRY_QUEUE.pollFirst()) != null) { + queued.flockReleaseRetryStarted.set(false); } } - }, "qdb-sf-flock-release-retry"); - retryThread.setDaemon(true); - retryThread.start(); + } + } + if (startFailure == null) { LOG.error("SF slot flock release failed during engine close; keeping " - + "closeCompleted=false and retrying outside the pool lock so " + + "closeCompleted=false and retrying on the shared driver so " + "retired capacity recovers after the transient failure [slot={}]", sfDir == null ? "" : sfDir); - } catch (Throwable t) { + } else { // A later explicit close() can still retry without repeating the - // one-time ring/watermark cleanup. The kernel remains the final - // process-exit backstop if thread creation and all retries fail. - flockReleaseRetryStarted.set(false); + // one-time ring/watermark cleanup. The failed queue is cleared so + // the driver does not retain engines it cannot service. LOG.error("Could not start SF flock-release retry driver; close() may be " + "invoked again to retry [slot={}, error={}]", - sfDir == null ? "" : sfDir, String.valueOf(t)); + sfDir == null ? "" : sfDir, String.valueOf(startFailure)); } } @@ -910,6 +1004,17 @@ public boolean isCloseCompleted() { return closeCompleted; } + /** + * Registers a callback for confirmed SF slot-lock release. If release + * already completed, invokes the callback before returning. + */ + public void setSlotLockReleaseListener(Runnable listener) { + slotLockReleaseListener = listener; + if (listener != null && closeCompleted) { + listener.run(); + } + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 13a69f77..32ec4b30 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -200,7 +200,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); - private WebSocketClient client; + private volatile WebSocketClient client; // Optional: when non-null, every server-rejection error (retriable and // terminal alike) is offered to the dispatcher for async delivery to the user's // handler. Null disables async delivery entirely; the producer-side @@ -763,6 +763,25 @@ public synchronized void close() { // would block forever. isAlive()==false also covers the normal // post-exit case where the latch is already counted down. if (t.isAlive()) { + // Break a native send/receive before joining. Full client close + // must remain after the worker exit because it frees buffers the + // worker may still access. + WebSocketClient activeClient = client; + if (activeClient != null) { + try { + activeClient.closeTraffic(); + } catch (Throwable e) { + // A custom transport that cannot safely cancel traffic + // must not be destructively closed while the worker is + // live. Fail without joining indefinitely; the worker's + // exit path retains ownership of final cleanup. + throw new LineSenderException( + "cursor I/O thread did not stop: active transport does not support safe traffic shutdown; " + + "client/engine teardown is delegated to the I/O thread's exit path", + e + ); + } + } try { shutdownLatch.await(); } catch (InterruptedException e) { @@ -1114,7 +1133,10 @@ private void clearDurableAckTracking() { * before the first connect attempt — see {@link #failPaced}. */ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptMillis) { - if (reconnectFactory == null || !running) { + if (!running) { + return; + } + if (reconnectFactory == null) { recordFatal(initial); return; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 15d5c607..578c4f07 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -80,6 +80,7 @@ public final class SegmentManager implements QuietCloseable { private static final long WORKER_JOIN_TIMEOUT_MILLIS = 5_000L; private final AtomicLong fileGeneration = new AtomicLong(); + private final FilesFacade filesFacade; private final Object lock = new Object(); private final long maxTotalBytes; // Reused by the manager worker thread to build spare-segment paths @@ -173,11 +174,11 @@ public final class SegmentManager implements QuietCloseable { private volatile Thread workerThread; public SegmentManager(long segmentSizeBytes) { - this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES); + this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE); } public SegmentManager(long segmentSizeBytes, long pollNanos) { - this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES); + this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE); } /** @@ -203,6 +204,11 @@ public SegmentManager(long segmentSizeBytes, long pollNanos) { * hold an initial active plus one hot spare. */ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) { + this(segmentSizeBytes, pollNanos, maxTotalBytes, FilesFacade.INSTANCE); + } + + @TestOnly + public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes, FilesFacade filesFacade) { // The pathScratch field initializer has already allocated its native // buffer by the time this body runs, so a validation throw must free // it or every failed construction leaks 256 bytes of native memory @@ -217,6 +223,7 @@ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) "maxTotalBytes (" + maxTotalBytes + ") must allow at least one segment of " + segmentSizeBytes + " bytes"); } + this.filesFacade = filesFacade; this.segmentSizeBytes = segmentSizeBytes; this.pollNanos = pollNanos; this.maxTotalBytes = maxTotalBytes; @@ -613,21 +620,19 @@ public void wakeWorker() { * files in {@code dir}, or {@code -1} if none exist. Skips files that * don't match the pattern (e.g. the legacy {@code sf-initial.sfa}). */ - private static long scanMaxGeneration(String dir) { + private long scanMaxGeneration(String dir) { long max = -1L; - if (!Files.exists(dir)) return max; - long find = Files.findFirst(dir); + if (!filesFacade.exists(dir)) return max; + long find = filesFacade.findFirst(dir); if (find < 0) { - LOG.warn("scanMaxGeneration could not enumerate {}; " - + "next spare may collide with an existing on-disk segment", dir); - return max; + throw new IllegalStateException("could not enumerate SF segment directory " + dir); } if (find == 0) return max; try { int rc = 1; while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); - rc = Files.findNext(find); + String name = Files.utf8ToString(filesFacade.findName(find)); + rc = filesFacade.findNext(find); if (name == null || !name.startsWith("sf-") || !name.endsWith(".sfa")) { continue; } @@ -640,8 +645,11 @@ private static long scanMaxGeneration(String dir) { // sf-initial.sfa or non-hex — skip } } + if (rc < 0) { + throw new IllegalStateException("could not fully enumerate SF segment directory " + dir); + } } finally { - Files.findClose(find); + filesFacade.findClose(find); } return max; } @@ -769,7 +777,7 @@ private void serviceRing0(RingEntry e) { // via its long-ptr overload, bypassing the byte[] + native // malloc that the String overload would incur on every // rotation. - spare = MmapSegment.create(FilesFacade.INSTANCE, + spare = MmapSegment.create(filesFacade, pathScratch.ptr(), path, e.ring.nextSeqHint(), segmentSizeBytes); } @@ -815,7 +823,7 @@ private void serviceRing0(RingEntry e) { // on disk, this is the second-line defense. Repeated // unlink on an already-removed path is a harmless no-op. if (path != null) { - Files.remove(path); + filesFacade.remove(path); } } } @@ -828,59 +836,54 @@ private void serviceRing0(RingEntry e) { // The watermark write and totalBytes commit are registration-gated // under `lock` so stale worker snapshots cannot touch the // engine-owned watermark or mutate accounting after deregister() - // returns. drainTrimmable still runs for stale snapshots: it - // transfers ownership of fully-acked sealed segments to this - // worker, preserving the old close + unlink behavior. + // returns. Trimming still runs for stale snapshots: a successful + // close + unlink transfers the fully-acked segment out of the ring, + // preserving cleanup without touching deregistered accounting. // // munmap + unlink stay outside the lock — they can be slow // and shouldn't block register/deregister or sibling rings. - ObjList trim; Runnable hook = beforeTrimSyncHook; if (hook != null) { hook.run(); } + boolean trimFailed = false; + while (true) { + MmapSegment segment = e.ring.firstTrimmable(); + if (segment == null) { + break; + } + String path = segment.path(); + try { + segment.close(); + if (path != null && !filesFacade.remove(path)) { + trimFailed = true; + LOG.warn("Failed to unlink trimmed segment {}", path); + break; + } + synchronized (lock) { + if (e.ring.removeTrimmable(segment) && e.isRegistered()) { + totalBytes -= segment.sizeBytes(); + } + } + } catch (Throwable t) { + trimFailed = true; + LOG.warn("Failed to trim segment {}", path == null ? "" : path, t); + break; + } + } + // An unlink failure leaves the acknowledged segment in the ring and + // on disk. Do not advance the persisted watermark past it: recovery + // must continue to see the failed path as live bookkeeping rather + // than infer that trim committed. A later successful retry persists + // the current cumulative ACK normally. synchronized (lock) { - boolean registered = e.isRegistered(); - // Persist the current ackedFsn watermark BEFORE the trim runs. - // On host crash between the persist and the unlinks below, the - // segments survive and the watermark is correct. On crash AFTER - // the unlinks but before next tick, the segments are gone and - // the watermark is stale, but recovery clamps with - // max(lowestSurvivingBaseSeq - 1, watermark) so either ordering - // is safe. Memory-mode rings (and callers that didn't supply a - // watermark) skip the write. - // Persist only on advance to avoid pointless mmap stores when - // ackedFsn is steady. The store is a single 8-byte put against - // an already-mapped region -- no syscall, no allocation -- but - // the gate keeps the dirty-page footprint minimal under - // steady-state load with no new acks arriving. - if (registered && e.watermark != null) { + if (!trimFailed && e.isRegistered() && e.watermark != null) { long currentAck = e.ring.ackedFsn(); if (currentAck > e.lastPersistedAck) { e.watermark.write(currentAck); e.lastPersistedAck = currentAck; } } - trim = e.ring.drainTrimmable(); - if (registered && trim != null) { - for (int i = 0, n = trim.size(); i < n; i++) { - totalBytes -= trim.get(i).sizeBytes(); - } - } - } - if (trim != null) { - for (int i = 0, n = trim.size(); i < n; i++) { - MmapSegment s = trim.get(i); - String path = s.path(); - try { - s.close(); - if (path != null && !Files.remove(path)) { - LOG.warn("Failed to unlink trimmed segment {}", path); - } - } catch (Throwable t) { - LOG.warn("Failed to trim segment {}", path == null ? "" : path, t); - } - } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index c8516c4c..8d929ae1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -516,6 +516,21 @@ public synchronized MmapSegment firstSealed() { return sealedSegments.size() > 0 ? sealedSegments.get(0) : null; } + /** + * Returns the oldest fully acknowledged sealed segment without removing + * it. The segment manager keeps it owned by the ring until close + unlink + * succeeds, so a failed unlink cannot make the path disappear from live + * bookkeeping or allow its identifier to be reused. + */ + public synchronized MmapSegment firstTrimmable() { + if (sealedSegments.size() == 0) { + return null; + } + MmapSegment segment = sealedSegments.get(0); + long lastSeq = segment.baseSeq() + segment.frameCount() - 1; + return lastSeq <= ackedFsn ? segment : null; + } + /** Active segment -- exposed for the I/O thread's "send next batch" path. */ /** * Walks every published frame in the ring (sealed segments plus the active @@ -634,6 +649,22 @@ public long publishedFsn() { return publishedFsn; } + /** + * Commits removal of the segment returned by {@link #firstTrimmable()}. + * Returns false if concurrent lifecycle activity changed the head. + */ + public synchronized boolean removeTrimmable(MmapSegment segment) { + if (sealedSegments.size() == 0 || sealedSegments.get(0) != segment) { + return false; + } + long lastSeq = segment.baseSeq() + segment.frameCount() - 1; + if (lastSeq > ackedFsn) { + return false; + } + sealedSegments.remove(0); + return true; + } + /** * Registers a wakeup callback that the producer thread will invoke when * a hot spare is needed -- either right after a rotation has consumed the diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index d2cae5ad..940e6049 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -220,8 +220,9 @@ public final class SenderPool implements AutoCloseable { // races it. recoveryInRangeNext is the next in-range index in [0, maxSize) // for pass 1; recoveryOutOfRange / recoveryOutOfRangeNext are the lazily // built pass-2 work list (same-base slots at index >= maxSize) and its - // cursor; recoveryComplete latches true when the whole scan finishes or is - // aborted, making runStartupRecoveryStep()/...ToCompletion() idempotent. + // cursor; recoveryComplete latches true only when the whole scan finishes. + // A transient build failure or drain timeout leaves the current candidate + // pending so a later tick or explicit drive can retry it on the same pool. private int recoveryInRangeNext; private IntList recoveryOutOfRange; private int recoveryOutOfRangeNext; @@ -427,8 +428,9 @@ void runStartupRecoveryToCompletion() { * No-op (returns {@code false}) when SF is off, the pool is shutting down, or * recovery has already finished. * - * @return {@code true} if recovery has more work (call again), {@code false} - * when recovery is complete or the pool is shutting down + * @return {@code true} if recovery has more work immediately; {@code false} + * when recovery is complete, the pool is shutting down, or a transient + * failure deferred the current candidate until a later tick */ boolean runStartupRecoveryStep() { if (!storeAndForward || closed || recoveryComplete) { @@ -469,9 +471,10 @@ boolean runStartupRecoveryStep() { * {@code slotInUse} entry and are never allocated by borrow(). *

    * Best-effort throughout: a build/close Error or a slow drain is logged and - * never propagates, since the data stays durable on disk for a later attempt; - * the first build failure or drain timeout latches {@code recoveryComplete} - * (the failure will very likely repeat for every remaining slot). + * never propagates, since the data stays durable on disk for a later attempt. + * A build failure or drain timeout stops the current drive but leaves its + * candidate pending so a later housekeeper tick can retry after a transient + * condition clears; it does not poison recovery for the life of the pool. *

    * Boundedness / residual window. Recovery is driven on the * PoolHousekeeper thread, and {@code close()} relies on a step finishing @@ -541,9 +544,8 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryInRangeNext++; continue; } - // A real candidate -> spend the step on it. Advance the cursor first - // so a resume never reprocesses this index. - recoveryInRangeNext++; + // A real candidate -> spend the step on it. Advance the cursor only + // after success so a transient build/drain failure remains retryable. boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, retained); lock.lock(); try { @@ -579,12 +581,12 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { lock.unlock(); } if (stopScan) { - // A build failure or drain timeout that will very likely repeat - // for every remaining slot -- abort the scan; the data stays - // durable on disk for a later attempt. Do not start pass 2. - recoveryComplete = true; + // Stop this drive without advancing the cursor. The same live + // pool retries this candidate after the transient condition is + // removed instead of requiring pool recreation. return false; } + recoveryInRangeNext++; return true; } @@ -605,9 +607,10 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { if (closed) { return false; } - int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext++); + int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext); String slotPath = sfDir + "/" + slotBaseId + "-" + idx; if (!OrphanScanner.isCandidateOrphan(slotPath)) { + recoveryOutOfRangeNext++; continue; } boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, retained); @@ -626,9 +629,12 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { slotPath); } if (stopScan) { - recoveryComplete = true; + // Keep the out-of-range cursor on this candidate. In particular, + // a transient flock/build collision must be retried after the + // primary sender returns; no capacity bookkeeping is involved. return false; } + recoveryOutOfRangeNext++; return true; } @@ -678,7 +684,7 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // likely repeat for every remaining slot, so stop here rather // than pay a connect timeout per slot. LOG.warn("startup SF recovery: could not open slot {} ({}); " - + "skipping remaining slots", slotPath, buildErr.toString()); + + "deferring this and remaining slots", slotPath, buildErr.toString()); return true; } try { @@ -688,7 +694,7 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // same reasoning as the build-failure case above. if (!recoverer.delegate().drain(remainingMillis)) { LOG.warn("startup SF recovery: drain did not ack slot {} " - + "within {}ms; skipping remaining slots", + + "within {}ms; deferring this and remaining slots", slotPath, remainingMillis); stopScan = true; } @@ -811,10 +817,10 @@ public PooledSender borrow() { // Capacity-starved: re-probe retired slots BEFORE the terminal // timeout check — a deferred engine cleanup may have released a // flock since the retire, and the freed index can admit a - // creation right now. The release itself never signals - // slotReleased (it happens in the delegate on a worker/I/O-thread - // exit path, volatile writes only), so this poll is the only way - // a borrower learns of it. Ordering matters twice over: a + // creation right now. The delegate normally signals this pool + // after deferred release, while this probe also covers delegates + // that do not expose that notification and release/listener races. + // Ordering matters twice over: a // zero-timeout (try-once) borrow must get its one probe before // throwing, and a borrower whose awaitNanos budget just expired // must get a final probe on its wake-up pass instead of timing @@ -1207,7 +1213,11 @@ public int leakedSlotCount() { } private SenderSlot createUnlocked(int slotIndex) { - return new SenderSlot(senderFactory.apply(slotIndex), this, slotIndex); + Sender delegate = senderFactory.apply(slotIndex); + if (delegate instanceof QwpWebSocketSender) { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(this::recoverReleasedSlots); + } + return new SenderSlot(delegate, this, slotIndex); } /** @@ -1218,7 +1228,11 @@ private SenderSlot createUnlocked(int slotIndex) { * {@link #drainCandidateSlotForRecovery}. */ private SenderSlot createRecoverer(int slotIndex) { - return new SenderSlot(recoverySenderFactory.apply(slotIndex), this, slotIndex); + Sender delegate = recoverySenderFactory.apply(slotIndex); + if (delegate instanceof QwpWebSocketSender) { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(this::recoverReleasedSlots); + } + return new SenderSlot(delegate, this, slotIndex); } private Sender defaultSender(int slotIndex) { @@ -1398,6 +1412,15 @@ private boolean reclaimSlot(SenderSlot s, String context) { return false; } + private void recoverReleasedSlots() { + lock.lock(); + try { + reprobeRetiredSlots(); + } finally { + lock.unlock(); + } + } + /** * Re-probes every retired slot (see {@link #reclaimSlot}) and returns to * the free set any whose delegate now reports the flock released — the diff --git a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java index c1b1eec7..b5e43a35 100644 --- a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java +++ b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java @@ -149,6 +149,14 @@ public void close() { } } + @Override + public void closeTraffic() { + // A concurrent TLS send may still use the SSLEngine and direct buffers. + // Shut down only the delegate traffic path here; close() releases the + // retained fd and frees TLS state after the owning worker has stopped. + delegate.closeTraffic(); + } + @Override public int getFd() { return delegate.getFd(); diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java index f649d330..b9e669d0 100644 --- a/core/src/main/java/io/questdb/client/network/Net.java +++ b/core/src/main/java/io/questdb/client/network/Net.java @@ -151,6 +151,8 @@ public static void init() { public static native int setTcpNoDelay(int fd, boolean noDelay); + public static native int shutdown(int fd); + public static long sockaddr(int ipv4address, int port) { SOCK_ADDR_COUNTER.incrementAndGet(); return sockaddr0(ipv4address, port); diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacade.java b/core/src/main/java/io/questdb/client/network/NetworkFacade.java index d23824a5..4c66f0df 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacade.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacade.java @@ -78,6 +78,15 @@ public interface NetworkFacade { int setTcpNoDelay(int fd, boolean noDelay); + /** + * Shuts down traffic on a descriptor owned by this facade without releasing + * the descriptor. Custom facades must override this method rather than let + * native code operate on a synthetic or remapped descriptor. + */ + default int shutdown(int fd) { + throw new UnsupportedOperationException("traffic shutdown is not supported by this network facade"); + } + long sockaddr(int address, int port); int socketTcp(boolean blocking); diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java index 64ea0dc7..1d1d7e20 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java @@ -132,6 +132,11 @@ public int setTcpNoDelay(int fd, boolean noDelay) { return Net.setTcpNoDelay(fd, noDelay); } + @Override + public int shutdown(int fd) { + return Net.shutdown(fd); + } + @Override public long sockaddr(int address, int port) { return Net.sockaddr(address, port); diff --git a/core/src/main/java/io/questdb/client/network/PlainSocket.java b/core/src/main/java/io/questdb/client/network/PlainSocket.java index 555affd2..cc175f51 100644 --- a/core/src/main/java/io/questdb/client/network/PlainSocket.java +++ b/core/src/main/java/io/questdb/client/network/PlainSocket.java @@ -29,7 +29,7 @@ public class PlainSocket implements Socket { private final Logger log; private final NetworkFacade nf; - private int fd = -1; + private volatile int fd = -1; public PlainSocket(NetworkFacade nf, Logger log) { this.nf = nf; @@ -37,13 +37,22 @@ public PlainSocket(NetworkFacade nf, Logger log) { } @Override - public void close() { + public synchronized void close() { if (fd != -1) { nf.close(fd, log); fd = -1; } } + @Override + public synchronized void closeTraffic() { + if (fd != -1 && nf.shutdown(fd) != 0) { + throw new IllegalStateException( + "could not shut down socket traffic [fd=" + fd + ", errno=" + nf.errno() + ']' + ); + } + } + @Override public int getFd() { return fd; diff --git a/core/src/main/java/io/questdb/client/network/Socket.java b/core/src/main/java/io/questdb/client/network/Socket.java index 0cdce517..6fceb30b 100644 --- a/core/src/main/java/io/questdb/client/network/Socket.java +++ b/core/src/main/java/io/questdb/client/network/Socket.java @@ -37,6 +37,17 @@ public interface Socket extends QuietCloseable { int WRITE_FLAG = 1; + /** + * Closes only the network traffic path so a concurrent blocking send or + * receive returns. Implementations must retain any I/O buffers until the + * owning worker has stopped and {@link #close()} performs full cleanup. + * The compatibility default fails without touching implementation-owned + * resources; custom sockets must override this method to opt in. + */ + default void closeTraffic() { + throw new UnsupportedOperationException("traffic shutdown is not supported by this socket"); + } + /** * @return file descriptor associated with the socket. */ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java index 59d0f83d..e4fb86c5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java @@ -51,6 +51,83 @@ */ public class CloseDrainTest { + @Test(timeout = 30_000L) + public void testCloseBlocksAcrossAllReplicaWindowUntilPromotion() throws Exception { + DelayingAckHandler handler = new DelayingAckHandler(0); + try (TestWebSocketServer server = new TestWebSocketServer(handler, false, "PRIMARY")) { + server.setRejectWithRole("REPLICA"); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + server.getPort() + + ";initial_connect_retry=async" + + ";reconnect_initial_backoff_millis=20" + + ";reconnect_max_backoff_millis=100" + + ";close_flush_timeout_millis=10000;"; + QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg); + CountDownLatch closeDrainWaiting = new CountDownLatch(1); + CountDownLatch releaseCloseDrain = new CountDownLatch(1); + AtomicReference closeFailure = new AtomicReference<>(); + AtomicReference hookFailure = new AtomicReference<>(); + sender.setCloseDrainWaitingHook(() -> { + closeDrainWaiting.countDown(); + try { + if (!releaseCloseDrain.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("promotion did not release the close-drain witness"); + } + } catch (Throwable t) { + hookFailure.set(t); + } + }); + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "all-replica-close-drain"); + try { + closer.start(); + Assert.assertTrue("server never produced the all-replica role rejection", + server.awaitRoleReject(5, TimeUnit.SECONDS)); + Assert.assertTrue("close never observed its real unacknowledged drain target", + closeDrainWaiting.await(5, TimeUnit.SECONDS)); + + Assert.assertTrue("pre-promotion witness must include a role rejection", + server.roleRejectCount() >= 1); + Assert.assertEquals("pre-promotion data must remain unacknowledged", + -1L, sender.getAckedFsn()); + Assert.assertTrue("close must remain pending for the whole all-replica window", + closer.isAlive()); + Assert.assertEquals("promotion must not have delivered or replayed the frame yet", + 0L, handler.nextSeq.get()); + + // The close-drain barrier has held continuously since it observed + // targetFsn > ackedFsn. Promote first, then release the barrier: + // completion therefore cannot precede the deterministic recovery event. + server.setAdvertisedRole("PRIMARY"); + server.setRejectWithRole(null); + releaseCloseDrain.countDown(); + closer.join(10_000L); + + Assert.assertFalse("close did not complete after promotion", closer.isAlive()); + Assert.assertNull("close-drain witness failed", hookFailure.get()); + Assert.assertNull("close failed after promotion", closeFailure.get()); + Assert.assertEquals("the unacknowledged frame must be delivered exactly once", + 1L, handler.nextSeq.get()); + } finally { + server.setAdvertisedRole("PRIMARY"); + server.setRejectWithRole(null); + releaseCloseDrain.countDown(); + closer.join(10_000L); + sender.close(); + } + } + } + @Test public void testCloseBlocksUntilAckArrives() throws Exception { // Server delays every ACK by 800ms. With the default diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java new file mode 100644 index 00000000..0a53e28e --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java @@ -0,0 +1,256 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class CursorWebSocketSendLoopBlockedSendCloseTest { + + @Test(timeout = 30_000L) + public void testCloseBreaksBlockedSendBeforeJoiningWorker() throws Exception { + TestUtils.assertMemoryLeak(() -> { + BlockingSendClient client = new BlockingSendClient(true); + CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + client, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + null, + 100L, + 1_000L, + 5_000L, + false + ); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + Thread closer = null; + AtomicReference closeFailure = new AtomicReference<>(); + try { + Unsafe.getUnsafe().putLong(payload, 0x0102030405060708L); + Unsafe.getUnsafe().putLong(payload + 8, 0x1112131415161718L); + Assert.assertEquals(0L, engine.appendBlocking(payload, 16)); + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking send", + client.sendEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "blocked-send-closer"); + closer.start(); + + Assert.assertTrue("close did not break the traffic path before joining", + client.trafficClosed.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("traffic path must close exactly once", 1, client.trafficCloseCount.get()); + Assert.assertEquals("the closer must break traffic, not the I/O worker", + closer, client.trafficCloseThread.get()); + Assert.assertTrue("blocked send did not observe traffic-path closure", + client.sendExited.await(5, TimeUnit.SECONDS)); + + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("close did not join the I/O worker", closer.isAlive()); + Assert.assertNull("close failed", closeFailure.get()); + Assert.assertNull("ordinary close must not manufacture a terminal error", loop.getTerminalError()); + + Thread ioThread = client.sendThread.get(); + Assert.assertNotNull(ioThread); + Assert.assertFalse("I/O worker must be dead when close returns", ioThread.isAlive()); + Assert.assertEquals("full client cleanup must run exactly once", 1, client.cleanupCount.get()); + Assert.assertEquals("the I/O worker must own cleanup before publishing exit", + ioThread, client.cleanupThread.get()); + Assert.assertFalse("close must not manufacture caller interruption", + closer.isInterrupted()); + } finally { + client.releaseSend.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + client.close(); + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test(timeout = 30_000L) + public void testUnsupportedCustomTransportFailsWithoutDestroyingWorkerResources() throws Exception { + TestUtils.assertMemoryLeak(() -> { + BlockingSendClient client = new BlockingSendClient(false); + CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + client, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + null, + 100L, + 1_000L, + 5_000L, + false + ); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + Thread closer = null; + AtomicReference closeFailure = new AtomicReference<>(); + CountDownLatch closeEntered = new CountDownLatch(1); + try { + Unsafe.getUnsafe().putLong(payload, 0x0102030405060708L); + Unsafe.getUnsafe().putLong(payload + 8, 0x1112131415161718L); + Assert.assertEquals(0L, engine.appendBlocking(payload, 16)); + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking send", + client.sendEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + closeEntered.countDown(); + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "unsupported-transport-closer"); + closer.start(); + + Assert.assertTrue("close did not start", closeEntered.await(5, TimeUnit.SECONDS)); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("unsupported transport made close join indefinitely", closer.isAlive()); + Assert.assertTrue(closeFailure.get() instanceof LineSenderException); + Assert.assertTrue(closeFailure.get().getCause() instanceof UnsupportedOperationException); + Assert.assertEquals("unsupported cancellation must not release the blocked send", + 1L, client.sendExited.getCount()); + Assert.assertEquals("unsupported cancellation must not perform full cleanup", 0, client.cleanupCount.get()); + Assert.assertTrue("worker must retain resource ownership", client.sendThread.get().isAlive()); + + client.releaseSend.countDown(); + Assert.assertTrue("released send did not exit", client.sendExited.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("worker did not complete delegated cleanup", client.cleanupDone.await(5, TimeUnit.SECONDS)); + client.sendThread.get().join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("worker lingered after the custom transport was released", client.sendThread.get().isAlive()); + Assert.assertEquals(1, client.cleanupCount.get()); + Assert.assertNull("ordinary worker exit must remain non-terminal", loop.getTerminalError()); + } finally { + client.releaseSend.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + Thread ioThread = client.sendThread.get(); + if (ioThread != null) { + ioThread.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + client.close(); + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static final class BlockingSendClient extends WebSocketClient { + private final AtomicBoolean cleanupClaimed = new AtomicBoolean(); + private final AtomicInteger cleanupCount = new AtomicInteger(); + private final CountDownLatch cleanupDone = new CountDownLatch(1); + private final AtomicReference cleanupThread = new AtomicReference<>(); + private final boolean trafficShutdownSupported; + private final CountDownLatch releaseSend = new CountDownLatch(1); + private final CountDownLatch sendEntered = new CountDownLatch(1); + private final CountDownLatch sendExited = new CountDownLatch(1); + private final AtomicReference sendThread = new AtomicReference<>(); + private final AtomicInteger trafficCloseCount = new AtomicInteger(); + private final AtomicReference trafficCloseThread = new AtomicReference<>(); + private final CountDownLatch trafficClosed = new CountDownLatch(1); + + private BlockingSendClient(boolean trafficShutdownSupported) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.trafficShutdownSupported = trafficShutdownSupported; + } + + @Override + public void close() { + if (cleanupClaimed.compareAndSet(false, true)) { + cleanupCount.incrementAndGet(); + cleanupThread.set(Thread.currentThread()); + cleanupDone.countDown(); + } + super.close(); + } + + @Override + public void closeTraffic() { + if (!trafficShutdownSupported) { + throw new UnsupportedOperationException("custom transport has no safe cancellation capability"); + } + trafficCloseThread.compareAndSet(null, Thread.currentThread()); + trafficCloseCount.incrementAndGet(); + trafficClosed.countDown(); + releaseSend.countDown(); + } + + @Override + public void sendBinary(long dataPtr, int length, int timeout) { + sendThread.set(Thread.currentThread()); + sendEntered.countDown(); + try { + if (!releaseSend.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("traffic path was not closed"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("I/O worker was interrupted instead of traffic being closed", e); + } finally { + sendExited.countDown(); + } + throw new LineSenderException("traffic path closed"); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java new file mode 100644 index 00000000..e6c0e143 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java @@ -0,0 +1,247 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FlockReleaseRetryDriverTest { + + private final List sfDirs = new ArrayList<>(); + + @After + public void tearDown() { + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + for (String sfDir : sfDirs) { + removeDir(sfDir); + } + } + + @Test(timeout = 30_000L) + public void testPersistentFailuresShareOneRetryThread() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int engineCount = 32; + AtomicInteger retryFailures = new AtomicInteger(); + AtomicInteger threadsCreated = new AtomicInteger(); + AtomicReference retryThreadRef = new AtomicReference<>(); + CountDownLatch retryFailuresObserved = new CountDownLatch(engineCount * 2); + CountDownLatch retryThreadStarted = new CountDownLatch(1); + CountDownLatch runRetryDriver = new CountDownLatch(1); + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(() -> { + retryFailures.incrementAndGet(); + retryFailuresObserved.countDown(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + threadsCreated.incrementAndGet(); + Thread thread = new Thread(() -> { + retryThreadStarted.countDown(); + try { + runRetryDriver.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + task.run(); + }, "test-shared-flock-release-retry"); + retryThreadRef.set(thread); + return thread; + }); + + List engines = new ArrayList<>(); + List slotLocks = new ArrayList<>(); + List realFds = new ArrayList<>(); + boolean fdsRestored = false; + try { + for (int i = 0; i < engineCount; i++) { + String sfDir = newSfDir("persistent-" + i); + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + engines.add(engine); + slotLocks.add(slotLock); + realFds.add(realFd); + setFd(slotLock, 1_000_000_000); + engine.close(); + assertFalse("injected unlock failure must keep close incomplete", + engine.isCloseCompleted()); + if (i == 0) { + assertTrue("retry thread was not started", + retryThreadStarted.await(10, TimeUnit.SECONDS)); + } + } + + assertEquals("persistent failures must share one retry thread", + 1, threadsCreated.get()); + runRetryDriver.countDown(); + assertTrue("driver did not perform two failed rounds", + retryFailuresObserved.await(10, TimeUnit.SECONDS)); + assertTrue("driver did not retain persistent failures", + retryFailures.get() >= engineCount * 2); + for (CursorSendEngine engine : engines) { + assertFalse("failed releases must remain unpublished", + engine.isCloseCompleted()); + } + + restoreFds(slotLocks, realFds); + fdsRestored = true; + Thread retryThread = retryThreadRef.get(); + retryThread.join(10_000L); + assertFalse("shared retry thread retained lifecycle resources after drain", + retryThread.isAlive()); + for (CursorSendEngine engine : engines) { + assertTrue("driver must release every restored flock", + engine.isCloseCompleted()); + } + assertEquals("retries must not create another thread", + 1, threadsCreated.get()); + } finally { + if (!fdsRestored) { + restoreFds(slotLocks, realFds); + } + runRetryDriver.countDown(); + Thread retryThread = retryThreadRef.get(); + if (retryThread != null) { + retryThread.join(10_000L); + } + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null); + } + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + @Test(timeout = 30_000L) + public void testRetryThreadStartFailureLeavesExplicitCloseRetryable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger starts = new AtomicInteger(); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> new Thread(task) { + @Override + public synchronized void start() { + starts.incrementAndGet(); + throw new IllegalStateException("injected start failure"); + } + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("start-failure"), 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + try { + setFd(slotLock, 1_000_000_000); + engine.close(); + assertEquals("retry driver start must be attempted once", 1, starts.get()); + assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted()); + + // This also proves the failed driver cleared its queue: the + // setter rejects replacement while any engine remains queued. + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + setFd(slotLock, realFd); + engine.close(); + assertTrue("explicit close must recover after retry-thread start failure", + engine.isCloseCompleted()); + } finally { + if (!engine.isCloseCompleted()) { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + setFd(slotLock, realFd); + if (!slotLock.release()) { + fail("restored flock fd did not release"); + } + } + } + }); + } + + private static int fd(SlotLock slotLock) throws Exception { + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + return fdField.getInt(slotLock); + } + + private static void removeDir(String sfDir) { + long find = Files.findFirst(sfDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(sfDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(sfDir); + } + + private static void restoreFds(List slotLocks, List realFds) throws Exception { + for (int i = 0; i < slotLocks.size(); i++) { + SlotLock slotLock = slotLocks.get(i); + synchronized (slotLock) { + setFd(slotLock, realFds.get(i)); + } + } + } + + private static void setFd(SlotLock slotLock, int fd) throws Exception { + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + fdField.setInt(slotLock, fd); + } + + private static SlotLock slotLock(CursorSendEngine engine) throws Exception { + Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock"); + slotLockField.setAccessible(true); + return (SlotLock) slotLockField.get(engine); + } + + private String newSfDir(String suffix) { + String sfDir = Paths.get( + System.getProperty("java.io.tmpdir"), + "qdb-flock-release-retry-" + suffix + "-" + System.nanoTime() + ).toString(); + sfDirs.add(sfDir); + return sfDir; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java new file mode 100644 index 00000000..0feffe8a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java @@ -0,0 +1,350 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class SegmentManagerUnlinkFailureTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-segmgr-unlink-fault-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir != null) { + removeRecursive(tmpDir); + } + } + + @Test + public void testEnumerationFindNextFailureRefusesGenerationAllocation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String dir = tmpDir + "/enumeration"; + Assert.assertEquals(0, Files.mkdir(dir, Files.DIR_MODE_DEFAULT)); + String lowerName = "sf-0000000000000000.sfa"; + String lowerPath = dir + "/" + lowerName; + String higherPath = dir + "/sf-0000000000000007.sfa"; + MmapSegment lower = MmapSegment.create(lowerPath, 0L, segmentSize); + lower.close(); + MmapSegment initial = MmapSegment.create(higherPath, 0L, segmentSize); + SegmentRing ring = new SegmentRing(initial, segmentSize); + byte[] originalHigher = java.nio.file.Files.readAllBytes(Paths.get(higherPath)); + FailingFilesFacade facade = new FailingFilesFacade(null, dir, lowerName); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 4, facade)) { + try { + manager.register(ring, dir); + Assert.fail("register accepted a partially enumerated SF directory"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("could not fully enumerate")); + } + Assert.assertTrue("fault did not occur after the lower generation was observed", + facade.partialLowerObserved); + Assert.assertTrue("failed enumeration cursor was not closed", facade.partialFindClosed); + Assert.assertEquals("enumeration failure must not allocate or truncate a path", + 0, facade.openCleanCalls); + Assert.assertTrue("higher generation disappeared", Files.exists(higherPath)); + Assert.assertArrayEquals("partial enumeration changed the unseen higher generation", + originalHigher, java.nio.file.Files.readAllBytes(Paths.get(higherPath))); + } finally { + ring.close(); + } + }); + } + + @Test(timeout = 15_000L) + public void testFailedUnlinkRetainsBookkeepingAndUsesSuccessorPath() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String dir = tmpDir + "/unlink"; + Assert.assertEquals(0, Files.mkdir(dir, Files.DIR_MODE_DEFAULT)); + String failedPath = dir + "/sf-0000000000000000.sfa"; + String activePath = dir + "/sf-0000000000000001.sfa"; + MmapSegment initial = MmapSegment.create(failedPath, 0L, segmentSize); + SegmentRing ring = new SegmentRing(initial, segmentSize); + AckWatermark watermark = AckWatermark.open(dir); + Assert.assertNotNull(watermark); + watermark.write(-1L); + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + try { + fill(payload, 32, (byte) 0x11); + Assert.assertEquals(0L, ring.appendOrFsn(payload, 32)); + ring.installHotSpare(MmapSegment.create(activePath, 1L, segmentSize)); + Assert.assertEquals(1L, ring.appendOrFsn(payload, 32)); + ring.acknowledge(0L); + byte[] original = java.nio.file.Files.readAllBytes(Paths.get(failedPath)); + + FailingFilesFacade facade = new FailingFilesFacade(failedPath, null, null); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8, facade)) { + manager.register(ring, dir, watermark); + manager.start(); + Assert.assertTrue("manager never attempted the injected unlink", + facade.removeAttempted.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("failed unlink must retain conservative registered bytes", + ring.totalSegmentBytes(), readTotalBytes(manager)); + } + + Assert.assertTrue("failed unlink path must remain observable", Files.exists(failedPath)); + Assert.assertTrue("failed unlink changed the acknowledged segment bytes", + Arrays.equals(original, java.nio.file.Files.readAllBytes(Paths.get(failedPath)))); + Assert.assertNotNull("failed unlink removed the segment from ring bookkeeping", + ring.firstSealed()); + Assert.assertEquals(failedPath, ring.firstSealed().path()); + Assert.assertEquals("watermark advanced although unlink did not commit", + -1L, watermark.read()); + + fill(payload, 32, (byte) 0x5A); + Assert.assertEquals("non-DEDUP successor must continue at the next FSN", + 2L, ring.appendOrFsn(payload, 32)); + String successorPath = ring.getActive().path(); + Assert.assertNotEquals("successor reused the acknowledged path", + failedPath, successorPath); + Assert.assertEquals(dir + "/sf-0000000000000002.sfa", successorPath); + Assert.assertTrue("successor segment was not created", Files.exists(successorPath)); + Assert.assertTrue("distinct non-DEDUP payload was not written to the successor", + containsRun(java.nio.file.Files.readAllBytes(Paths.get(successorPath)), + (byte) 0x5A, 32)); + Assert.assertTrue("successor write overwrote the failed-unlink segment", + Arrays.equals(original, java.nio.file.Files.readAllBytes(Paths.get(failedPath)))); + + try (SegmentManager retryManager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8, facade)) { + retryManager.register(ring, dir, watermark); + retryManager.start(); + retryManager.wakeWorker(); + awaitRetryCommit(failedPath, watermark); + } + MmapSegment firstSealed = ring.firstSealed(); + Assert.assertTrue("successful retry retained the acknowledged segment", + firstSealed == null || !failedPath.equals(firstSealed.path())); + } finally { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + ring.close(); + watermark.close(); + } + }); + } + + private static void awaitRetryCommit(String failedPath, AckWatermark watermark) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (Files.exists(failedPath) || watermark.read() != 0L) { + if (System.nanoTime() > deadline) { + throw new AssertionError("unlink retry did not remove the file and advance the watermark"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static boolean containsRun(byte[] bytes, byte value, int length) { + int run = 0; + for (byte b : bytes) { + run = b == value ? run + 1 : 0; + if (run == length) { + return true; + } + } + return false; + } + + private static void fill(long address, int len, byte value) { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(address + i, value); + } + } + + private static long readTotalBytes(SegmentManager manager) throws Exception { + Field field = SegmentManager.class.getDeclaredField("totalBytes"); + field.setAccessible(true); + Field lockField = SegmentManager.class.getDeclaredField("lock"); + lockField.setAccessible(true); + Object lock = lockField.get(manager); + synchronized (lock) { + return field.getLong(manager); + } + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static final class FailingFilesFacade implements FilesFacade { + private static final long PARTIAL_FIND_PTR = Long.MAX_VALUE; + private final String enumerationFailureDir; + private final String partialLowerName; + private final String unlinkFailurePath; + private final CountDownLatch removeAttempted = new CountDownLatch(1); + private int openCleanCalls; + private boolean partialFindClosed; + private long partialFindNamePtr; + private boolean partialLowerObserved; + private int unlinkFailuresRemaining = 1; + + private FailingFilesFacade( + String unlinkFailurePath, + String enumerationFailureDir, + String partialLowerName + ) { + this.unlinkFailurePath = unlinkFailurePath; + this.enumerationFailureDir = enumerationFailureDir; + this.partialLowerName = partialLowerName; + } + + @Override + public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override + public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override + public int close(int fd) { return INSTANCE.close(fd); } + @Override + public boolean exists(String path) { return INSTANCE.exists(path); } + @Override + public void findClose(long findPtr) { + if (findPtr == PARTIAL_FIND_PTR) { + INSTANCE.freeNativePath(partialFindNamePtr); + partialFindNamePtr = 0L; + partialFindClosed = true; + } else { + INSTANCE.findClose(findPtr); + } + } + @Override + public long findFirst(String dir) { + if (dir.equals(enumerationFailureDir)) { + partialFindNamePtr = INSTANCE.allocNativePath(partialLowerName); + return PARTIAL_FIND_PTR; + } + return INSTANCE.findFirst(dir); + } + @Override + public long findName(long findPtr) { + return findPtr == PARTIAL_FIND_PTR ? partialFindNamePtr : INSTANCE.findName(findPtr); + } + @Override + public int findNext(long findPtr) { + if (findPtr == PARTIAL_FIND_PTR) { + partialLowerObserved = true; + return -1; + } + return INSTANCE.findNext(findPtr); + } + @Override + public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override + public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override + public int fsync(int fd) { return INSTANCE.fsync(fd); } + @Override + public long length(int fd) { return INSTANCE.length(fd); } + @Override + public long length(String path) { return INSTANCE.length(path); } + @Override + public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override + public int lock(int fd) { return INSTANCE.lock(fd); } + @Override + public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int openCleanRW(String path) { + openCleanCalls++; + return INSTANCE.openCleanRW(path); + } + @Override + public int openCleanRW(long pathPtr) { + openCleanCalls++; + return INSTANCE.openCleanRW(pathPtr); + } + @Override + public int openRW(String path) { return INSTANCE.openRW(path); } + @Override + public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + @Override + public boolean remove(String path) { + if (path.equals(unlinkFailurePath) && unlinkFailuresRemaining > 0) { + unlinkFailuresRemaining--; + removeAttempted.countDown(); + return false; + } + return INSTANCE.remove(path); + } + @Override + public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override + public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override + public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 6d4c5ef0..139a445f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -64,6 +64,8 @@ public class TestWebSocketServer implements Closeable { // client-side pool actually closed the connections it opened. private final AtomicInteger liveConnections = new AtomicInteger(); private final int port; + private final AtomicInteger roleRejectCount = new AtomicInteger(); + private final CountDownLatch roleRejectLatch = new CountDownLatch(1); private final AtomicBoolean running = new AtomicBoolean(false); private final ServerSocket serverSocket; private final CountDownLatch startLatch = new CountDownLatch(1); @@ -165,6 +167,10 @@ public TestWebSocketServer(WebSocketServerHandler handler, this.port = serverSocket.getLocalPort(); } + public boolean awaitRoleReject(long timeout, TimeUnit unit) throws InterruptedException { + return roleRejectLatch.await(timeout, unit); + } + public boolean awaitStart(long timeout, TimeUnit unit) throws InterruptedException { return startLatch.await(timeout, unit); } @@ -221,6 +227,13 @@ public int liveConnectionCount() { return liveConnections.get(); } + /** + * Number of HTTP 421 role-reject responses sent over the server's lifetime. + */ + public int roleRejectCount() { + return roleRejectCount.get(); + } + /** * Replaces the advertised role for subsequent handshakes (live update). */ @@ -586,6 +599,8 @@ private boolean performHandshake() throws IOException { "\r\n"; out.write(sb.getBytes(StandardCharsets.US_ASCII)); out.flush(); + roleRejectCount.incrementAndGet(); + roleRejectLatch.countDown(); return false; } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 6db9e009..896dc69a 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -777,6 +777,100 @@ public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { }); } + @Test(timeout = 30_000) + public void testDeferredFlockReleaseWakesParkedLongTimeoutBorrower() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 1, 1, 60_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender lease = pool.borrow(); + Sender delegate = getDelegate(lease); + CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); + SlotLock slotLock = (SlotLock) getField(engine, "slotLock"); + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(slotLock); + Assert.assertTrue("precondition: live flock fd", realFd >= 0); + + CountDownLatch borrowerAcquired = new CountDownLatch(1); + CountDownLatch borrowerParked = new CountDownLatch(1); + CountDownLatch releaseBorrower = new CountDownLatch(1); + AtomicReference borrowerFailure = new AtomicReference<>(); + AtomicReference recovered = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + recovered.set(pool.borrow()); + borrowerAcquired.countDown(); + if (!releaseBorrower.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to release borrower"); + } + } catch (Throwable t) { + borrowerFailure.compareAndSet(null, t); + } finally { + PooledSender sender = recovered.get(); + if (sender != null) { + sender.close(); + } + } + }, "sender-pool-deferred-release-waiter"); + + try { + synchronized (slotLock) { + fdField.setInt(slotLock, 1_000_000_000); + } + invokeDiscardBroken(pool, lease); + Assert.assertEquals("failed release must retire the only slot", + 1, pool.leakedSlotCount()); + Assert.assertFalse(engine.isCloseCompleted()); + + pool.setBeforeBorrowWaitHook(borrowerParked::countDown); + borrower.start(); + Assert.assertTrue("borrower must reach the condition wait with a long timeout", + borrowerParked.await(5, TimeUnit.SECONDS)); + // The hook runs under the pool lock immediately before awaitNanos. + // Acquiring that lock here proves awaitNanos atomically enqueued the + // borrower and released the lock before restoration can start. This + // read-only operation neither changes pool state nor signals a waiter. + pool.availableSize(); + + // This restored fd is the only source of progress: the retry driver + // confirms the release. There is no housekeeper or pool mutation. + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + } + Assert.assertTrue("deferred flock release must wake the parked borrower", + borrowerAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertNull("borrower must not fail", borrowerFailure.get()); + Assert.assertEquals("release wakeup must recover retired capacity", + 0, pool.leakedSlotCount()); + } finally { + pool.setBeforeBorrowWaitHook(null); + releaseBorrower.countDown(); + if (!engine.isCloseCompleted()) { + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + } + } + borrower.join(TimeUnit.SECONDS.toMillis(1)); + if (borrower.isAlive()) { + borrower.interrupt(); + borrower.join(TimeUnit.SECONDS.toMillis(5)); + } + Assert.assertFalse("borrower thread must finish", borrower.isAlive()); + } + if (borrowerFailure.get() != null) { + throw new AssertionError("borrower failed", borrowerFailure.get()); + } + } + } + }); + } + @Test public void testMixedRetiredSlotsRecoverWithoutLosingAccounting() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -2201,6 +2295,84 @@ public void testDefaultConfigRecoversOutOfRangeSlotsAfterShrink() throws Excepti }); } + @Test + public void testFailedOutOfRangeRecoveryRetriesAfterPrimaryReturns() throws Exception { + // A deferred pool can already have its sole in-range sender borrowed when + // startup recovery reaches an out-of-range slot left by a larger pool. + // A transient recoverer build failure must leave that candidate pending: + // after the primary lease returns, the SAME pool must retry and drain it. + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-1;close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", 1L).atNow(); + seed.flush(); + } + } + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicBoolean primaryReturned = new AtomicBoolean(); + AtomicInteger recoveryAttempts = new AtomicInteger(); + IntFunction factory = idx -> { + if (idx == 1) { + recoveryAttempts.incrementAndGet(); + if (!primaryReturned.get()) { + throw new LineSenderException("transient out-of-range recovery failure"); + } + } + return Sender.builder(config).senderId("default-" + idx).build(); + }; + + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { + PooledSender primary = pool.borrow(); + Object primarySlot = slotOf(primary); + + Assert.assertFalse("first recovery attempt must stop at the transient failure", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals("exactly one out-of-range recovery attempt", 1, + recoveryAttempts.get()); + Assert.assertTrue("failed recovery must preserve the candidate", + hasSegmentFile(slot("default-1"))); + Assert.assertEquals("out-of-range failure must not consume in-range capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("out-of-range recoverer must not enter retired-slot bookkeeping", + ((List) getField(pool, "retiredSlots")).isEmpty()); + + primary.close(); + primaryReturned.set(true); + invokeRunStartupRecoveryOnce(pool); + + Assert.assertEquals("same live pool must retry the out-of-range candidate", 2, + recoveryAttempts.get()); + Assert.assertFalse("retry must drain the preserved out-of-range data", + hasSegmentFile(slot("default-1"))); + Assert.assertTrue("retry must deliver the recovered frame", handler.frames.get() >= 1); + Assert.assertEquals("successful out-of-range retry must not consume capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("out-of-range retry must leave retired slots untouched", + ((List) getField(pool, "retiredSlots")).isEmpty()); + + PooledSender next = pool.borrow(); + try { + Assert.assertSame("normal borrow must reuse the returned primary slot", + primarySlot, slotOf(next)); + } finally { + next.close(); + } + } + } + }); + } + @Test public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws Exception { // The drain exclusion is bounded to [0, maxSize) so a sibling's drainer diff --git a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java new file mode 100644 index 00000000..47d168f9 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java @@ -0,0 +1,463 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.network; + +import io.questdb.client.network.JavaTlsClientSocketFactory; +import io.questdb.client.network.Kqueue; +import io.questdb.client.network.KqueueFacade; +import io.questdb.client.network.KqueueFacadeImpl; +import io.questdb.client.network.NetworkFacade; +import io.questdb.client.network.NetworkFacadeImpl; +import io.questdb.client.network.PlainSocket; +import io.questdb.client.network.Socket; +import io.questdb.client.network.SocketReadinessWaiter; +import io.questdb.client.network.TlsSessionInitFailedException; +import io.questdb.client.std.Os; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class SocketTrafficShutdownTest { + private static final NetworkFacade NF = NetworkFacadeImpl.INSTANCE; + + private static class CompatibilityNetworkFacade implements NetworkFacade { + private final AtomicInteger closeCount; + + private CompatibilityNetworkFacade(AtomicInteger closeCount) { + this.closeCount = closeCount; + } + + @Override + public int close(int fd) { + closeCount.incrementAndGet(); + return 0; + } + + @Override + public void close(int fd, org.slf4j.Logger logger) { + close(fd); + } + + @Override + public void configureKeepAlive(int fd) { + } + + @Override + public int configureNonBlocking(int fd) { + return 0; + } + + @Override + public int connect(int fd, long pSockaddr) { + return 0; + } + + @Override + public int connectAddrInfo(int fd, long pAddrInfo) { + return 0; + } + + @Override + public int connectAddrInfoTimeout(int fd, long pAddrInfo, int timeoutMillis) { + return 0; + } + + @Override + public int errno() { + return 0; + } + + @Override + public void freeAddrInfo(long pAddrInfo) { + } + + @Override + public void freeSockAddr(long pSockaddr) { + } + + @Override + public long getAddrInfo(CharSequence host, int port) { + return 0; + } + + @Override + public int getSndBuf(int fd) { + return 0; + } + + @Override + public int recvRaw(int fd, long buffer, int bufferLen) { + return 0; + } + + @Override + public int sendRaw(int fd, long buffer, int bufferLen) { + return 0; + } + + @Override + public int sendToRaw(int fd, long lo, int len, long socketAddress) { + return 0; + } + + @Override + public int sendToRawScatter(int fd, long segmentsPtr, int segmentCount, long socketAddress) { + return 0; + } + + @Override + public int setMulticastInterface(int fd, int ipv4Address) { + return 0; + } + + @Override + public int setMulticastTtl(int fd, int ttl) { + return 0; + } + + @Override + public boolean setSndBuf(int fd, int size) { + return false; + } + + @Override + public int setTcpNoDelay(int fd, boolean noDelay) { + return 0; + } + + @Override + public long sockaddr(int address, int port) { + return 0; + } + + @Override + public int socketTcp(boolean blocking) { + return 0; + } + + @Override + public int socketUdp() { + return 0; + } + + @Override + public boolean testConnection(int fd, long buffer, int bufferSize) { + return false; + } + } + + private static class CompatibilitySocket implements Socket { + private final AtomicInteger closeCount; + + private CompatibilitySocket(AtomicInteger closeCount) { + this.closeCount = closeCount; + } + + @Override + public void close() { + closeCount.incrementAndGet(); + } + + @Override + public int getFd() { + return -1; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void of(int fd) { + } + + @Override + public int recv(long bufferPtr, int bufferLen) { + return -1; + } + + @Override + public int send(long bufferPtr, int bufferLen) { + return -1; + } + + @Override + public void startTlsSession(CharSequence peerName, SocketReadinessWaiter waiter) throws TlsSessionInitFailedException { + } + + @Override + public boolean supportsTls() { + return false; + } + + @Override + public int tlsIO(int readinessFlags) { + return 0; + } + + @Override + public boolean wantsTlsWrite() { + return false; + } + } + + @Test + public void testCompatibilityDefaultsDoNotBypassCustomTransportOwnership() { + AtomicInteger facadeCloseCount = new AtomicInteger(); + NetworkFacade customFacade = new CompatibilityNetworkFacade(facadeCloseCount); + + PlainSocket plainSocket = new PlainSocket(customFacade, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + plainSocket.of(42); + assertUnsupported(plainSocket::closeTraffic); + Assert.assertEquals("facade compatibility fallback must not release a synthetic descriptor", + 0, facadeCloseCount.get()); + Assert.assertEquals(42, plainSocket.getFd()); + plainSocket.close(); + Assert.assertEquals(1, facadeCloseCount.get()); + + AtomicInteger socketCloseCount = new AtomicInteger(); + Socket customSocket = new CompatibilitySocket(socketCloseCount); + assertUnsupported(customSocket::closeTraffic); + Assert.assertEquals("socket compatibility fallback must not run destructive close", + 0, socketCloseCount.get()); + } + + @Test(timeout = 30_000L) + public void testPlainSocketShutdownWakesMacOsKqueueAndRetainsFd() throws Exception { + assertShutdownWakesMacOsKqueue(new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class))); + } + + @Test + public void testTlsSocketTrafficGatePreservesTlsStateUntilFullClose() throws Exception { + AtomicInteger closeCount = new AtomicInteger(); + AtomicInteger shutdownCount = new AtomicInteger(); + NetworkFacade facade = (NetworkFacade) Proxy.newProxyInstance( + NetworkFacade.class.getClassLoader(), + new Class[]{NetworkFacade.class}, + (proxy, method, args) -> { + if ("close".equals(method.getName())) { + closeCount.incrementAndGet(); + return method.getReturnType() == int.class ? 0 : null; + } + if ("shutdown".equals(method.getName())) { + shutdownCount.incrementAndGet(); + return 0; + } + if (method.getReturnType() == boolean.class) { + return false; + } + if (method.getReturnType() == int.class) { + return 0; + } + if (method.getReturnType() == long.class) { + return 0L; + } + return null; + } + ); + Socket socket = JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION.newInstance( + facade, + LoggerFactory.getLogger(SocketTrafficShutdownTest.class) + ); + socket.of(42); + Field sslEngineField = socket.getClass().getDeclaredField("sslEngine"); + Field stateField = socket.getClass().getDeclaredField("state"); + sslEngineField.setAccessible(true); + stateField.setAccessible(true); + sslEngineField.set(socket, SSLContext.getDefault().createSSLEngine()); + stateField.setInt(socket, 2); // JavaTlsClientSocket.STATE_TLS + Object[] tlsState = snapshotTlsState(socket); + + try { + socket.closeTraffic(); + + Object[] stateAfterTrafficClose = snapshotTlsState(socket); + for (int i = 0; i < tlsState.length; i++) { + if (i == 2) { + Assert.assertEquals("traffic cancellation must preserve TLS state", tlsState[i], stateAfterTrafficClose[i]); + } else { + Assert.assertSame("traffic cancellation must preserve SSLEngine/buffer references", + tlsState[i], stateAfterTrafficClose[i]); + } + } + Assert.assertEquals(1, shutdownCount.get()); + Assert.assertEquals("traffic cancellation must not release the delegate fd", 0, closeCount.get()); + Assert.assertEquals(42, socket.getFd()); + Assert.assertFalse(socket.isClosed()); + } finally { + // Restore a valid plaintext state so full close does not attempt a + // synthetic TLS close_notify with uninitialised session buffers. + sslEngineField.set(socket, null); + stateField.setInt(socket, 1); // JavaTlsClientSocket.STATE_PLAINTEXT + socket.close(); + } + Assert.assertEquals(1, closeCount.get()); + Assert.assertTrue(socket.isClosed()); + } + + @Test(timeout = 30_000L) + public void testTlsSocketTrafficGateUsesDelegateShutdownAndRetainsFd() throws Exception { + assertShutdownWakesMacOsKqueue(JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION.newInstance( + NF, + LoggerFactory.getLogger(SocketTrafficShutdownTest.class) + )); + } + + private static void assertShutdownWakesMacOsKqueue(Socket socket) throws Exception { + Assume.assumeTrue("real kqueue cancellation coverage runs on macOS", Os.type == Os.DARWIN); + + AtomicBoolean pollDone = new AtomicBoolean(); + AtomicInteger pollResult = new AtomicInteger(Integer.MIN_VALUE); + AtomicReference pollFailure = new AtomicReference<>(); + CountDownLatch pollEntered = new CountDownLatch(1); + KqueueFacade facade = new KqueueFacade() { + private final KqueueFacade delegate = KqueueFacadeImpl.INSTANCE; + + @Override + public NetworkFacade getNetworkFacade() { + return delegate.getNetworkFacade(); + } + + @Override + public int kevent(int kq, long changeList, int nChanges, long eventList, int nEvents, int timeout) { + if (eventList != 0 && nEvents > 0) { + pollEntered.countDown(); + } + return delegate.kevent(kq, changeList, nChanges, eventList, nEvents, timeout); + } + + @Override + public int kqueue() { + return delegate.kqueue(); + } + }; + + int fd = -1; + Thread waiter = null; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfoTimeout(fd, addrInfo, 5_000)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept(); Kqueue kqueue = new Kqueue(facade, 1)) { + try { + Assert.assertEquals(0, NF.configureNonBlocking(fd)); + socket.of(fd); + int retainedFd = fd; + fd = -1; + + kqueue.setWriteOffset(0); + kqueue.readFD(retainedFd, 0); + Assert.assertEquals(0, kqueue.register(1)); + + waiter = new Thread(() -> { + try { + pollResult.set(kqueue.poll(10_000)); + } catch (Throwable t) { + pollFailure.set(t); + } finally { + pollDone.set(true); + } + }, "socket-traffic-kqueue-waiter"); + waiter.start(); + + Assert.assertTrue("waiter did not enter the native kqueue wait", + pollEntered.await(5, TimeUnit.SECONDS)); + Assert.assertFalse("peer unexpectedly made the read wait ready", pollDone.get()); + + socket.closeTraffic(); + + waiter.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("shutdown did not wake the native kqueue wait", waiter.isAlive()); + Assert.assertNull("native kqueue wait failed", pollFailure.get()); + Assert.assertTrue("shutdown must produce a readiness event", pollResult.get() > 0); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + } finally { + socket.closeTraffic(); + if (waiter != null) { + waiter.join(TimeUnit.SECONDS.toMillis(5)); + } + socket.close(); + } + } + } finally { + if (fd != -1) { + NF.close(fd); + } + } + } + + private static void assertUnsupported(Runnable operation) { + try { + operation.run(); + Assert.fail("expected unsupported traffic shutdown"); + } catch (UnsupportedOperationException expected) { + // Expected compatibility behavior: no native or destructive fallback. + } + } + + private static Object[] snapshotTlsState(Socket socket) throws Exception { + String[] fieldNames = { + "callerOutputBuffer", + "sslEngine", + "state", + "unwrapInputBuffer", + "unwrapOutputBuffer", + "wrapInputBuffer", + "wrapOutputBuffer" + }; + Object[] state = new Object[fieldNames.length]; + for (int i = 0; i < fieldNames.length; i++) { + Field field = socket.getClass().getDeclaredField(fieldNames[i]); + field.setAccessible(true); + state[i] = field.get(socket); + } + return state; + } +} From cad1bb8d27565381bb8965630b46beb74f2a8570 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 12 Jul 2026 00:34:05 +0100 Subject: [PATCH 16/26] fix(qwp): make close-time SF cleanup crash-safe and bound flock-release retries Close-time segment cleanup (review finding: failed cleanup published as reusable slot): - finishClose now persists the final acked FSN through the still-mapped watermark before closing the ring and before any unlink, so any cleanup failure or crash leaves the residue covered by a current watermark - unlinkAllSegmentFiles enumerates fully first, aborts with no unlinks on a failed directory walk, removes segments in ascending generation order and stops at the first failure, so residue is always a contiguous top slice that recovery seeds as fully acked (no replay, no duplicates) - the ack watermark is removed only after every segment is confirmed gone Flock-release retry driver (review finding: unbounded retry threads): - the shared retry driver now backs off exponentially per fully-failed round (100ms base, 5s cap), resets on progress, and is unparked when a fresh engine enqueues - after an injected driver-start failure, pool probes re-arm the retry via ensureFlockReleaseRetryScheduled() from isSlotLockReleased(), so retained capacity recovers without a second explicit close() New regression tests: CursorSendEngineCloseUnlinkFailureTest (close-time unlink fault injection; successor must not see replayable frames) and three FlockReleaseRetryDriverTest cases (backoff schedule, reset-on-progress, pool-probe recovery after start failure). --- .../qwp/client/QwpWebSocketSender.java | 24 +- .../client/sf/cursor/CursorSendEngine.java | 192 +++++++++++-- .../io/questdb/client/impl/SenderPool.java | 7 +- ...ursorSendEngineCloseUnlinkFailureTest.java | 209 ++++++++++++++ .../cursor/FlockReleaseRetryDriverTest.java | 261 ++++++++++++++++++ 5 files changed, 661 insertions(+), 32 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 9f454c9c..403748bf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1357,19 +1357,29 @@ public void close() { * manager-worker quiescence or I/O-thread exit path, this re-probes the * retained engine and latches true the moment that cleanup completes — pools re-probe retired * slots through this getter to recover their capacity. Monotonic: - * false→true only, never back. Lock-free (volatile reads) so pools may - * call it under their capacity lock. + * false→true only, never back. Cheap (volatile reads on every common + * path) so pools may call it under their capacity lock; only the rare + * orphaned-retry state below does more. + *

    + * The probe is also the recovery surface for a retained engine whose + * flock-release retry fell off the shared driver because the driver + * thread could not start (e.g. OOM at thread creation): close() is + * one-shot, so without the re-arm below that slot's capacity would stay + * lost until process exit. */ public boolean isSlotLockReleased() { if (slotLockReleased) { return true; } CursorSendEngine engine = retainedEngine; - if (engine != null && engine.isCloseCompleted()) { - // Benign latch race: concurrent callers may both observe the - // completed cleanup and both write true. - slotLockReleased = true; - return true; + if (engine != null) { + if (engine.isCloseCompleted()) { + // Benign latch race: concurrent callers may both observe the + // completed cleanup and both write true. + slotLockReleased = true; + return true; + } + engine.ensureFlockReleaseRetryScheduled(); } return false; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 95addd39..01910fcb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -31,9 +31,11 @@ import org.jetbrains.annotations.TestOnly; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; +import java.util.function.LongConsumer; /** * Facade that bundles a {@link SegmentRing} with a {@link SegmentManager} and @@ -69,10 +71,13 @@ public final class CursorSendEngine implements QuietCloseable { org.slf4j.LoggerFactory.getLogger(CursorSendEngine.class); private static final ThreadFactory DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY = runnable -> new Thread(runnable, "qdb-sf-flock-release-retry"); + private static final long FLOCK_RELEASE_RETRY_BASE_PARK_NANOS = 100_000_000L; // 100 ms private static final Object FLOCK_RELEASE_RETRY_LOCK = new Object(); + private static final long FLOCK_RELEASE_RETRY_MAX_PARK_NANOS = 5_000_000_000L; // 5 s private static final ArrayDeque FLOCK_RELEASE_RETRY_QUEUE = new ArrayDeque<>(); private static volatile Runnable afterFlockReleaseRetryFailureHook; private static volatile Runnable beforeDeferredCloseCreationHook; + private static volatile LongConsumer flockReleaseRetryParkOverride; private static Thread flockReleaseRetryThread; private static volatile ThreadFactory flockReleaseRetryThreadFactory = DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY; @@ -742,6 +747,27 @@ public synchronized void close() { */ private void finishClose(boolean fullyDrained) { try { + // On a fully-drained close, persist the final acked FSN through + // the still-mapped watermark BEFORE closing the ring/watermark + // and BEFORE unlinking any segment file. The manager persists + // the watermark only on its own tick, so it may lag the final + // ack. If the unlink below then fails (or the process dies + // mid-unlink), residual acknowledged .sfa files without an + // up-to-date watermark would seed the successor's recovery at + // lowestBase - 1 and replay already-acknowledged rows -- + // duplicates on a non-DEDUP table. The write is a single mmap + // store, so it succeeds even when the unlink is about to fail + // (e.g. the slot dir turned read-only). Quiescence is already + // established here, so no manager tick can race this write. + if (fullyDrained && watermark != null) { + try { + long finalAckedFsn = ring.ackedFsn(); + if (finalAckedFsn >= 0) { + watermark.write(finalAckedFsn); + } + } catch (Throwable ignored) { + } + } try { ring.close(); } catch (Throwable ignored) { @@ -761,13 +787,26 @@ private void finishClose(boolean fullyDrained) { } } if (fullyDrained) { + boolean segmentsRemoved = false; try { - unlinkAllSegmentFiles(sfDir); + segmentsRemoved = unlinkAllSegmentFiles(sfDir); } catch (Throwable ignored) { } - try { - AckWatermark.removeOrphan(sfDir); - } catch (Throwable ignored) { + // Remove the watermark ONLY once every segment file is + // confirmed gone. The watermark is what keeps residual + // acknowledged segments inert to a successor's recovery; + // removing it while any .sfa file survives would republish + // those already-acknowledged rows. + if (segmentsRemoved) { + try { + AckWatermark.removeOrphan(sfDir); + } catch (Throwable ignored) { + } + } else { + LOG.warn("close-time segment cleanup incomplete on slot {}; retaining the ack " + + "watermark so residual acknowledged segments stay covered -- the next " + + "engine on this slot recovers them as fully acked and retries the " + + "unlink on its own close", sfDir); } } } finally { @@ -813,6 +852,17 @@ public static void setBeforeDeferredCloseCreationHook(Runnable hook) { beforeDeferredCloseCreationHook = hook; } + /** + * Replaces the shared retry driver's inter-round park with a callback + * receiving the park duration the driver would have used. Test-only: + * makes the retry cadence inspectable and rounds coordinatable without + * wall-clock waits. + */ + @TestOnly + public static void setFlockReleaseRetryParkOverride(LongConsumer override) { + flockReleaseRetryParkOverride = override; + } + /** * Overrides creation of the single shared flock-release retry thread. * Test-only: makes thread creation/start failure and persistent retry @@ -903,6 +953,11 @@ private boolean retryFlockReleaseIfReady() { } private static void runFlockReleaseRetryDriver() { + // Capped exponential backoff: a persistent unlock failure must not + // burn a fixed 10 rounds of native release syscalls per second + // forever, but a transient failure must still recover promptly. The + // ramp doubles per fully-failed round from 100 ms up to 5 s. + long parkNanos = FLOCK_RELEASE_RETRY_BASE_PARK_NANOS; while (true) { final int roundSize; synchronized (FLOCK_RELEASE_RETRY_LOCK) { @@ -913,6 +968,7 @@ private static void runFlockReleaseRetryDriver() { } } boolean hasFailures = false; + boolean hasSuccesses = false; for (int i = 0; i < roundSize; i++) { final CursorSendEngine engine; synchronized (FLOCK_RELEASE_RETRY_LOCK) { @@ -920,6 +976,7 @@ private static void runFlockReleaseRetryDriver() { } if (engine.retryFlockReleaseIfReady()) { engine.flockReleaseRetryStarted.set(false); + hasSuccesses = true; } else { synchronized (FLOCK_RELEASE_RETRY_LOCK) { FLOCK_RELEASE_RETRY_QUEUE.addLast(engine); @@ -931,11 +988,22 @@ private static void runFlockReleaseRetryDriver() { } } } + if (hasSuccesses) { + // Progress: the failure condition is clearing, so retry the + // remaining engines on the base cadence again. + parkNanos = FLOCK_RELEASE_RETRY_BASE_PARK_NANOS; + } if (hasFailures) { // Interruption must not abandon a retained flock, but clear // the flag so subsequent parks still throttle retries. Thread.interrupted(); - LockSupport.parkNanos(100_000_000L); + LongConsumer parkOverride = flockReleaseRetryParkOverride; + if (parkOverride != null) { + parkOverride.accept(parkNanos); + } else { + LockSupport.parkNanos(parkNanos); + } + parkNanos = Math.min(parkNanos * 2, FLOCK_RELEASE_RETRY_MAX_PARK_NANOS); } } } @@ -947,7 +1015,12 @@ private void startFlockReleaseRetry() { Throwable startFailure = null; synchronized (FLOCK_RELEASE_RETRY_LOCK) { FLOCK_RELEASE_RETRY_QUEUE.addLast(this); - if (flockReleaseRetryThread == null) { + if (flockReleaseRetryThread != null) { + // The driver may be parked on a ramped backoff; wake it so a + // freshly failed release gets its first driver retry promptly + // instead of inheriting older engines' full backoff. + LockSupport.unpark(flockReleaseRetryThread); + } else { try { Thread retryThread = flockReleaseRetryThreadFactory.newThread( CursorSendEngine::runFlockReleaseRetryDriver); @@ -973,11 +1046,13 @@ private void startFlockReleaseRetry() { + "retired capacity recovers after the transient failure [slot={}]", sfDir == null ? "" : sfDir); } else { - // A later explicit close() can still retry without repeating the - // one-time ring/watermark cleanup. The failed queue is cleared so - // the driver does not retain engines it cannot service. - LOG.error("Could not start SF flock-release retry driver; close() may be " - + "invoked again to retry [slot={}, error={}]", + // A later explicit close() or a pool's retired-slot probe + // (ensureFlockReleaseRetryScheduled) can still retry without + // repeating the one-time ring/watermark cleanup. The failed queue + // is cleared so the driver does not retain engines it cannot + // service. + LOG.error("Could not start SF flock-release retry driver; a retried close() " + + "or pool re-probe re-arms the retry [slot={}, error={}]", sfDir == null ? "" : sfDir, String.valueOf(startFailure)); } } @@ -1015,6 +1090,25 @@ public void setSlotLockReleaseListener(Runnable listener) { } } + /** + * Re-arms the shared flock-release retry for an engine whose terminal + * cleanup finished but whose confirmed flock release is still pending + * and no longer scheduled — the retry driver thread failed to start when + * the release first failed (e.g. OOM at thread creation). + * {@code Sender.close()} is one-shot by contract, so pool probes + * ({@code QwpWebSocketSender.isSlotLockReleased()}) call this to keep a + * retired slot's capacity recoverable instead of lost until process + * exit. Cheap unless the engine is in that orphan state (volatile reads, + * then one failed CAS when the retry is already scheduled), so probes + * may call it under their capacity lock. + */ + public void ensureFlockReleaseRetryScheduled() { + if (closeCompleted || !terminalResourcesCleaned) { + return; + } + startFlockReleaseRetry(); + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ @@ -1112,37 +1206,91 @@ public long recoveredOrphanTipFsn() { return recoveredOrphanTipFsn; } + /** + * Ascending removal rank of a segment file name for + * {@link #unlinkAllSegmentFiles(String)}. {@code sf-initial.sfa} is + * always the fresh-start segment at baseSeq 0, so it ranks first. + * Spare files carry a monotonic generation ({@code sf-.sfa}) + * assigned in creation == rotation == baseSeq order, so the parsed + * generation ranks them. Anything else with the extension is not a + * live segment and ranks last. + */ + private static long segmentCleanupRank(String name) { + if ("sf-initial.sfa".equals(name)) { + return Long.MIN_VALUE; + } + if (name.length() == 23 && name.startsWith("sf-")) { + try { + return Long.parseUnsignedLong(name.substring(3, 19), 16); + } catch (NumberFormatException ignored) { + // fall through to the unrecognized-name rank + } + } + return Long.MAX_VALUE; + } + /** * Unlinks every {@code .sfa} file under {@code dir}. Called only on * clean shutdown when the ring confirms every published FSN has been * acked — at that moment the slot has no recoverable work and the * files are pure noise that would mislead the next sender's recovery. - * Best-effort: logs and continues on failures, since we're already on - * the close path. + *

    + * Removal runs in ascending segment order and STOPS at the first + * failure, so whatever survives (a failure here, or a crash mid-loop) + * is always a contiguous top slice of the ring: recovery's + * FSN-contiguity check still passes, and the retained ack watermark + * (== the final acked FSN == the highest frame on disk) still covers + * every surviving frame, so a successor replays nothing. + * + * @return {@code true} only when enumeration succeeded and every + * {@code .sfa} file was confirmed removed — the caller keeps the ack + * watermark on {@code false} so residual acknowledged segments stay + * covered. */ - private static void unlinkAllSegmentFiles(String dir) { - if (!io.questdb.client.std.Files.exists(dir)) return; + private static boolean unlinkAllSegmentFiles(String dir) { + if (!io.questdb.client.std.Files.exists(dir)) return true; long find = io.questdb.client.std.Files.findFirst(dir); if (find < 0) { LOG.warn("close-time unlink could not enumerate {}; " + "any residual sf-*.sfa files will be picked up by the next recovery", dir); - return; + return false; } - if (find == 0) return; + if (find == 0) return true; + ArrayList names = new ArrayList<>(); + int rc = 1; try { - int rc = 1; while (rc > 0) { String name = io.questdb.client.std.Files.utf8ToString( io.questdb.client.std.Files.findName(find)); rc = io.questdb.client.std.Files.findNext(find); - if (name == null || !name.endsWith(".sfa")) continue; - String path = dir + "/" + name; - if (!io.questdb.client.std.Files.remove(path)) { - LOG.warn("Failed to unlink fully-acked segment {} on close", path); + if (name != null && name.endsWith(".sfa")) { + names.add(name); } } } finally { io.questdb.client.std.Files.findClose(find); } + if (rc < 0) { + // A partial listing must not drive any unlink: removing only the + // files we happened to see could delete the segment holding the + // highest frame while a lower one survives, leaving residual + // state the retained watermark can no longer vouch for. + LOG.warn("close-time unlink could not fully enumerate {}; " + + "leaving all segment files for the next recovery", dir); + return false; + } + names.sort((a, b) -> { + int byRank = Long.compare(segmentCleanupRank(a), segmentCleanupRank(b)); + return byRank != 0 ? byRank : a.compareTo(b); + }); + for (int i = 0, n = names.size(); i < n; i++) { + String path = dir + "/" + names.get(i); + if (!io.questdb.client.std.Files.remove(path)) { + LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the " + + "residual files stay a contiguous, watermark-covered range", path); + return false; + } + } + return true; } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 940e6049..e401f864 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1428,9 +1428,10 @@ private void recoverReleasedSlots() { * run since the retire. Restores {@code leakedSlots} capacity and signals * waiters so a parked borrow can admit a creation immediately. *

    - * Caller must hold {@code lock}. The probe is lock-free on the delegate - * side ({@code isSlotLockReleased()} reads volatiles only), so holding - * the pool lock across it cannot stall behind delegate teardown. + * Caller must hold {@code lock}. The probe is cheap on the delegate side + * ({@code isSlotLockReleased()} reads volatiles, and only re-arms the + * shared flock-release retry in the rare orphaned-retry state), so + * holding the pool lock across it cannot stall behind delegate teardown. * * @return {@code true} if at least one slot's capacity was recovered */ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java new file mode 100644 index 00000000..a9df2b67 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java @@ -0,0 +1,209 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Regression for the close-time segment cleanup on a fully-drained slot. + *

    + * {@code CursorSendEngine.finishClose} unlinks the acknowledged {@code .sfa} + * files and then removes the ack watermark. When the unlink fails (transient + * I/O error, permission problem), the residual segment files hold rows the + * server already acknowledged. If the watermark does not cover them, a + * successor engine on the same slot seeds recovery from {@code lowestBase - 1} + * and replays every acknowledged row — duplicates on a non-DEDUP table. + *

    + * The test injects the unlink failure by dropping write permission on the + * slot directory (POSIX: unlink requires a writable parent directory), so it + * is skipped on Windows and when permissions are not enforced (root). + * The shared {@link SegmentManager} is deliberately never started: no worker + * thread exists, so no manager tick can persist the watermark behind the + * test's back, and the close-path quiescence barrier is trivially satisfied — + * fully deterministic, no timing coordination needed. + */ +public class CursorSendEngineCloseUnlinkFailureTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-close-unlink-fault-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir != null) { + removeRecursive(tmpDir); + } + } + + @Test(timeout = 20_000L) + public void testFailedCloseTimeUnlinkMustNotExposeAckedFramesToSuccessor() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String slot = tmpDir + "/slot"; + Path slotPath = Paths.get(slot); + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60)); + CursorSendEngine pred = null; + CursorSendEngine succ = null; + boolean slotDirReadOnly = false; + try { + fill(payload, 32, (byte) 0x33); + pred = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertEquals(0L, pred.appendBlocking(payload, 32)); + Assert.assertEquals(0L, pred.publishedFsn()); + // The server durably acknowledged FSN 0 in this session. + Assert.assertTrue(pred.acknowledge(0L)); + Assert.assertEquals(0L, pred.ackedFsn()); + + // Inject a close-time unlink failure: drop write permission on + // the slot dir. Prove the injection works with a probe file -- + // root (and some filesystems) ignore directory permissions. + String probePath = slot + "/probe"; + Assert.assertTrue(java.nio.file.Files.exists( + java.nio.file.Files.createFile(Paths.get(probePath)))); + try { + setPermissions(slotPath, "r-xr-xr-x"); + } catch (UnsupportedOperationException e) { + Assume.assumeNoException("POSIX permissions unavailable on this platform", e); + } + slotDirReadOnly = true; + boolean probeRemoved = Files.remove(probePath); + if (probeRemoved) { + setPermissions(slotPath, "rwxr-xr-x"); + slotDirReadOnly = false; + } + Assume.assumeFalse("directory permissions not enforced (running as root?)", + probeRemoved); + + // Fully-drained close: tries to unlink the acknowledged + // segment files and fails. + pred.close(); + Assert.assertTrue("flock release needs no dir write; close must complete", + pred.isCloseCompleted()); + pred = null; + Assert.assertTrue("injected unlink failure must leave the acknowledged segment", + Files.exists(slot + "/sf-initial.sfa")); + + // The transient failure clears before the successor arrives. + setPermissions(slotPath, "rwxr-xr-x"); + slotDirReadOnly = false; + Files.remove(probePath); + + succ = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertTrue(succ.wasRecoveredFromDisk()); + Assert.assertEquals(0L, succ.publishedFsn()); + // THE regression: FSN 0 was acknowledged by the server during + // the predecessor's session. The successor must not see it as + // replayable, or a non-DEDUP table receives duplicate rows. + Assert.assertTrue("successor exposes already-acknowledged frames for replay " + + "[ackedFsn=" + succ.ackedFsn() + + ", publishedFsn=" + succ.publishedFsn() + "]", + succ.ackedFsn() >= succ.publishedFsn()); + + // The successor's own fully-drained close retries the cleanup + // now that the failure has cleared: segments and watermark gone. + succ.close(); + Assert.assertTrue(succ.isCloseCompleted()); + succ = null; + Assert.assertFalse("successor close did not retry the segment unlink", + Files.exists(slot + "/sf-initial.sfa")); + Assert.assertFalse("watermark must be removed once no segment file remains", + Files.exists(slot + "/" + AckWatermark.FILE_NAME)); + } finally { + if (slotDirReadOnly) { + try { + setPermissions(slotPath, "rwxr-xr-x"); + } catch (Throwable ignored) { + } + } + if (pred != null) { + pred.close(); + } + if (succ != null) { + succ.close(); + } + manager.close(); + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static void fill(long address, int len, byte value) { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(address + i, value); + } + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static void setPermissions(Path path, String posix) throws Exception { + Set perms = PosixFilePermissions.fromString(posix); + java.nio.file.Files.setPosixFilePermissions(path, perms); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java index e6c0e143..0037437f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.std.Files; @@ -36,6 +37,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -52,12 +54,98 @@ public class FlockReleaseRetryDriverTest { @After public void tearDown() { CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null); + CursorSendEngine.setFlockReleaseRetryParkOverride(null); CursorSendEngine.setFlockReleaseRetryThreadFactory(null); for (String sfDir : sfDirs) { removeDir(sfDir); } } + /** + * Driver-start failure must not strand retired capacity until process + * exit. {@code Sender.close()} is a one-shot no-op by contract, so the + * only recovery surface a pool has is its retired-slot probe + * ({@code isSlotLockReleased()}, called from the housekeeper tick and + * capacity-starved borrows) — that probe must re-arm the shared retry + * driver once thread creation works again. + */ + @Test(timeout = 30_000L) + public void testDriverStartFailureRecoversViaPoolProbe() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger starts = new AtomicInteger(); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> new Thread(task) { + @Override + public synchronized void start() { + starts.incrementAndGet(); + throw new IllegalStateException("injected start failure"); + } + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("probe-rearm"), 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setCursorEngine(engine, true); + AtomicReference rearmedDriver = new AtomicReference<>(); + boolean recovered = false; + try { + setFd(slotLock, 1_000_000_000); + sender.close(); + assertEquals("retry driver start must be attempted once", 1, starts.get()); + assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted()); + + // Sender.close() is idempotent: repeat calls never reach the + // engine again, so they cannot restart the failed driver. + sender.close(); + assertEquals("no-op close must not retry the driver start", 1, starts.get()); + + // While the fault persists, each pool probe re-attempts the + // re-arm (and fails again) without publishing a release. + assertFalse("failed unlock must keep the slot reported as held", + sender.isSlotLockReleased()); + assertTrue("pool probe must re-attempt the driver start", + starts.get() >= 2); + + // The transient condition clears: thread creation works again + // (the failed start drained the queue, so the factory swap is + // legal) and the flock fd is back. + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-rearmed-flock-release-retry"); + rearmedDriver.set(thread); + return thread; + }); + setFd(slotLock, realFd); + CountDownLatch released = new CountDownLatch(1); + engine.setSlotLockReleaseListener(released::countDown); + + // Production recovery surface: the pool re-probes the retired + // slot through isSlotLockReleased(). + sender.isSlotLockReleased(); + assertTrue("pool probe must re-arm the flock-release retry after " + + "a driver start failure", + released.await(10, TimeUnit.SECONDS)); + assertTrue(engine.isCloseCompleted()); + assertTrue("probe must expose the recovered release", + sender.isSlotLockReleased()); + recovered = true; + } finally { + Thread driver = rearmedDriver.get(); + if (driver != null) { + driver.join(10_000L); + } + if (!recovered) { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + setFd(slotLock, realFd); + if (!slotLock.release()) { + fail("restored flock fd did not release"); + } + } + } + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + @Test(timeout = 30_000L) public void testPersistentFailuresShareOneRetryThread() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -149,6 +237,179 @@ public void testPersistentFailuresShareOneRetryThread() throws Exception { }); } + /** + * Schedule inspection for the shared driver's retry cadence: the + * inter-round park must grow exponentially from 100 ms and cap at 5 s, + * so a persistent unlock failure does not burn 10 syscalls per second + * per engine forever. The park override replaces the real park, so the + * test coordinates rounds without wall-clock waits. + */ + @Test(timeout = 30_000L) + public void testRetryBackoffDoublesToCap() throws Exception { + TestUtils.assertMemoryLeak(() -> { + List parks = new ArrayList<>(); + Semaphore parked = new Semaphore(0); + Semaphore proceed = new Semaphore(0); + AtomicReference driverRef = new AtomicReference<>(); + CursorSendEngine.setFlockReleaseRetryParkOverride(nanos -> { + parks.add(nanos); + parked.release(); + proceed.acquireUninterruptibly(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-backoff-flock-release-retry"); + driverRef.set(thread); + return thread; + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("backoff-cap"), 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + boolean fdRestored = false; + try { + setFd(slotLock, 1_000_000_000); + engine.close(); + assertFalse("injected unlock failure must keep close incomplete", + engine.isCloseCompleted()); + + // Eight failed rounds: enough to observe the full ramp and + // two capped parks. + for (int round = 1; round <= 8; round++) { + assertTrue("driver did not park after failed round " + round, + parked.tryAcquire(10, TimeUnit.SECONDS)); + if (round < 8) { + proceed.release(); + } + } + + setFd(slotLock, realFd); + fdRestored = true; + proceed.release(); + Thread driver = driverRef.get(); + driver.join(10_000L); + assertFalse("driver did not drain after the release succeeded", + driver.isAlive()); + assertTrue("restored flock must be released", engine.isCloseCompleted()); + + List expected = new ArrayList<>(); + expected.add(100_000_000L); + expected.add(200_000_000L); + expected.add(400_000_000L); + expected.add(800_000_000L); + expected.add(1_600_000_000L); + expected.add(3_200_000_000L); + expected.add(5_000_000_000L); + expected.add(5_000_000_000L); + assertEquals("retry parks must double from 100ms and cap at 5s", + expected, parks); + } finally { + if (!fdRestored) { + setFd(slotLock, realFd); + } + proceed.release(1_000); + Thread driver = driverRef.get(); + if (driver != null) { + driver.join(10_000L); + } + } + CursorSendEngine.setFlockReleaseRetryParkOverride(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + /** + * A successful release in a round is progress: the driver must reset its + * backoff to the base so the remaining engines are retried promptly + * while the failure condition is clearing. + */ + @Test(timeout = 30_000L) + public void testRetryBackoffResetsOnProgress() throws Exception { + TestUtils.assertMemoryLeak(() -> { + List parks = new ArrayList<>(); + Semaphore parked = new Semaphore(0); + Semaphore proceed = new Semaphore(0); + AtomicReference driverRef = new AtomicReference<>(); + CursorSendEngine.setFlockReleaseRetryParkOverride(nanos -> { + parks.add(nanos); + parked.release(); + proceed.acquireUninterruptibly(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-reset-flock-release-retry"); + driverRef.set(thread); + return thread; + }); + + CursorSendEngine engineA = new CursorSendEngine( + newSfDir("backoff-reset-a"), 4L * 1024 * 1024); + CursorSendEngine engineB = new CursorSendEngine( + newSfDir("backoff-reset-b"), 4L * 1024 * 1024); + SlotLock slotLockA = slotLock(engineA); + SlotLock slotLockB = slotLock(engineB); + int realFdA = fd(slotLockA); + int realFdB = fd(slotLockB); + boolean fdARestored = false; + boolean fdBRestored = false; + try { + setFd(slotLockA, 1_000_000_000); + setFd(slotLockB, 1_000_000_001); + engineA.close(); + engineB.close(); + + // Three failed rounds ramp the backoff to 400ms. + for (int round = 1; round <= 3; round++) { + assertTrue("driver did not park after failed round " + round, + parked.tryAcquire(10, TimeUnit.SECONDS)); + if (round < 3) { + proceed.release(); + } + } + + // Engine A recovers; round 4 has one success and one failure, + // so its park must be back at the 100ms base. + setFd(slotLockA, realFdA); + fdARestored = true; + proceed.release(); + assertTrue("driver did not park after the mixed round", + parked.tryAcquire(10, TimeUnit.SECONDS)); + assertTrue("recovered engine must publish completion", + engineA.isCloseCompleted()); + + setFd(slotLockB, realFdB); + fdBRestored = true; + proceed.release(); + Thread driver = driverRef.get(); + driver.join(10_000L); + assertFalse("driver did not drain after both releases succeeded", + driver.isAlive()); + assertTrue(engineB.isCloseCompleted()); + + List expected = new ArrayList<>(); + expected.add(100_000_000L); + expected.add(200_000_000L); + expected.add(400_000_000L); + expected.add(100_000_000L); + assertEquals("a successful release must reset the backoff to base", + expected, parks); + } finally { + if (!fdARestored) { + setFd(slotLockA, realFdA); + } + if (!fdBRestored) { + setFd(slotLockB, realFdB); + } + proceed.release(1_000); + Thread driver = driverRef.get(); + if (driver != null) { + driver.join(10_000L); + } + } + CursorSendEngine.setFlockReleaseRetryParkOverride(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + @Test(timeout = 30_000L) public void testRetryThreadStartFailureLeavesExplicitCloseRetryable() throws Exception { TestUtils.assertMemoryLeak(() -> { From a531164472c722c2ca5a771161eb0ae92fc8c039 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 12 Jul 2026 20:32:05 +0100 Subject: [PATCH 17/26] fix(qwp): harden SF crash recovery and cleanup Make acknowledged segment deletion crash-consistent, bound retry and cleanup paths, improve pool recovery complexity, and add deterministic lifecycle/native regression coverage. --- core/src/main/c/share/files.c | 15 + core/src/main/c/windows/files.c | 19 + .../qwp/client/QwpWebSocketSender.java | 263 ++++----- .../qwp/client/sf/cursor/AckWatermark.java | 70 ++- .../client/sf/cursor/CursorSendEngine.java | 59 +- .../sf/cursor/CursorWebSocketSendLoop.java | 63 +-- .../qwp/client/sf/cursor/MmapSegment.java | 19 + .../qwp/client/sf/cursor/SegmentManager.java | 297 ++++++++-- .../qwp/client/sf/cursor/SegmentRing.java | 157 ++++-- .../sf/cursor/SenderConnectionDispatcher.java | 6 + .../sf/cursor/SenderErrorDispatcher.java | 6 + .../sf/cursor/SenderProgressDispatcher.java | 6 + .../io/questdb/client/impl/SenderPool.java | 166 +++++- .../io/questdb/client/impl/SenderSlot.java | 13 + .../client/std/DefaultFilesFacade.java | 10 + .../java/io/questdb/client/std/Files.java | 16 + .../io/questdb/client/std/FilesFacade.java | 8 + .../java/io/questdb/client/std/ObjList.java | 2 +- .../client/SlotLockReleasedContractTest.java | 129 ++++- .../CursorSendEngineCrashConsistencyTest.java | 310 ++++++++++ ...CursorSendEngineSlotReacquisitionTest.java | 32 +- .../SegmentManagerCrashConsistencyTest.java | 529 ++++++++++++++++++ .../client/sf/cursor/SegmentManagerTest.java | 76 ++- .../SegmentManagerTrimDeregisterRaceTest.java | 8 +- .../SegmentManagerUnlinkFailureTest.java | 4 +- .../qwp/client/sf/cursor/SegmentRingTest.java | 140 ++++- .../test/impl/SenderPoolErrorSafetyTest.java | 107 ++++ .../client/test/impl/SenderPoolSfTest.java | 191 +++++++ .../network/SocketTrafficShutdownTest.java | 86 +++ .../questdb/client/test/std/ObjListTest.java | 17 + 30 files changed, 2464 insertions(+), 360 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java diff --git a/core/src/main/c/share/files.c b/core/src/main/c/share/files.c index 209c4490..fefd24fb 100644 --- a/core/src/main/c/share/files.c +++ b/core/src/main/c/share/files.c @@ -124,6 +124,21 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsync return res; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsyncDir0 + (JNIEnv *e, jclass cl, jlong lpszName) { + int fd; + RESTARTABLE(open((const char *) (uintptr_t) lpszName, O_RDONLY), fd); + if (fd < 0) { + return -1; + } + int res; + RESTARTABLE(fsync(fd), res); + int saved_errno = errno; + close(fd); + errno = saved_errno; + return res; +} + JNIEXPORT jboolean JNICALL Java_io_questdb_client_std_Files_truncate (JNIEnv *e, jclass cl, jint fd, jlong size) { int res; diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index d1d3b413..620bd81e 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -215,6 +215,25 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsync return 0; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsyncDir0 + (JNIEnv *e, jclass cl, jlong lpszName) { + jint fd = open_file((const char *) (uintptr_t) lpszName, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS); + if (fd < 0) { + return -1; + } + if (!FlushFileBuffers(FD_TO_HANDLE(fd))) { + SaveLastError(); + CloseHandle(FD_TO_HANDLE(fd)); + return -1; + } + CloseHandle(FD_TO_HANDLE(fd)); + return 0; +} + JNIEXPORT jboolean JNICALL Java_io_questdb_client_std_Files_truncate (JNIEnv *e, jclass cl, jint fd, jlong size) { FILE_END_OF_FILE_INFO eof; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 403748bf..8300cedc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -195,6 +195,8 @@ public class QwpWebSocketSender implements Sender { // close() drain timeout in millis. Default applied at construction. // 0 or -1 means "fast close" (skip the drain); otherwise close blocks // up to this many millis for ackedFsn to catch up to publishedFsn. + private volatile boolean closeCleanupComplete; + private boolean closeCleanupStarted; private long closeFlushTimeoutMillis = 5_000L; private volatile boolean closed; // Test-only lifecycle witness. close() invokes and clears it strictly after @@ -1206,136 +1208,26 @@ public void close() { } if (!ioThreadStopped) { - // The I/O thread may still be using the socket and microbatch - // buffers. Freeing them would risk SIGSEGV. - LOG.error("I/O thread is still running, leaking WebSocket client and microbatch buffers"); - // The engine, however, need not leak: delegate its close to - // the I/O thread's exit path, which runs it strictly after - // the thread's last engine access — the mapping and slot - // lock release as soon as the stuck wire call resolves - // (bounded by OS timeouts). slotLockReleased intentionally - // stays false: the lock is released only when the delegated - // close actually runs, so the pool must not reuse the slot - // meanwhile. A false return means the thread exited between - // the failed close() and now — then closing here is safe. - if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null) { - if (cursorSendLoop.delegateEngineClose()) { - // The I/O thread adopted the engine close and runs it on - // its exit path. Keep the engine visible for re-probing: - // isSlotLockReleased() flips true once that close - // completes, letting a pool recover the retired slot. - retainedEngine = cursorEngine; - } else { - CursorSendEngine engine = cursorEngine; - try { - engine.close(); - } catch (Throwable t) { - LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - if (engine.isCloseCompleted()) { - cursorEngine = null; - ownsCursorEngine = false; - slotLockReleased = true; - } else { - // Engine cleanup is pending on its manager worker's - // exit path (or leaked, for a shared manager). Report - // the retained flock now; the getter re-probes the - // retained engine so a late release is not lost. - slotLockReleased = false; - retainedEngine = engine; - } - } + // The worker may still touch every resource below. Hand the + // complete sender-owned tail to its exit path rather than + // permanently leaking everything except the engine. The + // callback is idempotence-gated by closeRemainingResources(). + if (ownsCursorEngine && cursorEngine != null) { + retainedEngine = cursorEngine; } - rethrowTerminal(terminalError); - return; - } - - if (buffer0 != null) { - try { - buffer0.close(); - } catch (Throwable t) { - LOG.error("Error closing buffer0: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (buffer1 != null) { - try { - buffer1.close(); - } catch (Throwable t) { - LOG.error("Error closing buffer1: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - - if (client != null) { - try { - client.close(); - } catch (Throwable t) { - LOG.error("Error closing WebSocket client: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - client = null; - } - - if (ownsCursorEngine && cursorEngine != null) { - CursorSendEngine engine = cursorEngine; - try { - engine.close(); - } catch (Throwable t) { - LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - if (engine.isCloseCompleted()) { - cursorEngine = null; - ownsCursorEngine = false; - slotLockReleased = true; - } else { - // The manager worker did not quiesce. Preserve ownership - // and report the retained flock so pools retire this slot. - // Repeated Sender.close() calls remain no-ops by contract. - // Engine cleanup was handed to a safe manager-worker path: - // owned-manager exit or shared-manager ring-pass completion. - // The getter re-probes the retained engine so the pool can - // reclaim the slot once cleanup actually completes. - slotLockReleased = false; - retainedEngine = engine; + Runnable closeCallback = () -> closeRemainingResources(null); + if (cursorSendLoop != null && cursorSendLoop.delegateClose(closeCallback)) { + rethrowTerminal(terminalError); + return; } + // The worker exited between close() failing and delegation. + // Cleanup is safe here and its failures remain suppressed on + // the original close error. + terminalError = closeRemainingResources(terminalError); } else { - // This sender owns no cursor engine holding an SF flock. - slotLockReleased = true; + terminalError = closeRemainingResources(terminalError); } - // Shutdown order: dispatcher last, after the I/O loop has stopped - // producing into it. close() drains pending entries with a short - // deadline so any final errors land in the user's handler. - if (errorDispatcher != null) { - try { - errorDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing error dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (progressDispatcher != null) { - try { - progressDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing progress dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (connectionDispatcher != null) { - try { - connectionDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing connection dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - - LOG.info("QwpWebSocketSender closed"); - // If close() ended up holding the same instance the user already // caught earlier, suppress the rethrow. The user's catch block // wraps close() (try-with-resources), and Throwable refuses @@ -1347,6 +1239,11 @@ public void close() { } } + @TestOnly + public boolean isCloseCleanupComplete() { + return closeCleanupComplete; + } + /** * True once the store-and-forward slot flock has been released. False * means an I/O or manager worker did not stop and close() retained the @@ -1808,6 +1705,21 @@ public long getDroppedConnectionNotifications() { return d == null ? 0L : d.getDroppedNotifications(); } + @TestOnly + public SenderConnectionDispatcher getConnectionDispatcherForTesting() { + return connectionDispatcher; + } + + @TestOnly + public SenderErrorDispatcher getErrorDispatcherForTesting() { + return errorDispatcher; + } + + @TestOnly + public SenderProgressDispatcher getProgressDispatcherForTesting() { + return progressDispatcher; + } + /** * Number of {@link SenderError} notifications dropped because the * bounded inbox was full. Non-zero means the user-supplied @@ -2234,6 +2146,11 @@ public void setClientFactoryOverride(java.util.function.Supplier keys) { int tableCount = 0; for (int i = 0, n = keys.size(); i < n; i++) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index 469c91ea..6e7ebc8c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; @@ -49,11 +50,11 @@ * offset 8: i64 fsn * *

    - * Zero-alloc, zero-syscall writes: {@link #open(String)} opens the + * Zero-alloc, store-only ACK writes: {@link #open(String)} opens the * file and maps the 16 bytes once. {@link #write(long)} is a single - * 8-byte aligned {@code Unsafe.putLong} into the mapped region. No - * malloc/free, no read/write syscalls, no rename, on the manager's hot - * tick path. An 8-byte aligned store is hardware-atomic on x86_64 and + * 8-byte aligned {@code Unsafe.putLong} into the mapped region. Ordinary + * ACK-only manager updates require no malloc/free, read/write syscalls, or + * rename. An 8-byte aligned store is hardware-atomic on x86_64 and * arm64, and disk-atomic within one sector (the file is 16 bytes — * trivially within one sector), so a torn FSN across a crash boundary * is not a concern. @@ -67,11 +68,13 @@ * complexity (multi-store with fence + read-side validate) for * marginal additional safety. *

    - * fsync cadence: intentionally NOT performed. Host crash falls - * back to recovery's {@code lowestBase - 1} seed (same as before this - * feature, no regression), at the cost of a bounded re-replay window - * for whatever durable-acks landed since the last successful page - * cache flush. + * fsync cadence: ordinary ACK-only manager updates call + * {@link #write(long)} and stay syscall-free. Each non-empty background disk-trim + * quantum calls {@link #sync()} once (one mmap msync and one fd fsync), fsyncs + * the slot directory before unlinking, and fsyncs it again after the batch. A + * fully drained close uses the same + * covering order so the durable watermark guards any acknowledged segment that + * a host crash restores. *

    * Lifecycle: single-writer (the {@link SegmentManager} worker * thread) after construction. Read once at engine startup (any thread, @@ -98,6 +101,7 @@ public final class AckWatermark implements QuietCloseable { private static final Logger LOG = LoggerFactory.getLogger(AckWatermark.class); private static final int MAGIC_OFFSET = 0; private final int fd; + private final FilesFacade filesFacade; private final long mmapAddress; private boolean closed; // Stamped once per process either at open() (if the file already @@ -108,8 +112,9 @@ public final class AckWatermark implements QuietCloseable { // construction; no synchronisation needed. private boolean magicWritten; - private AckWatermark(int fd, long mmapAddress, boolean magicAlreadyWritten) { + private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress, boolean magicAlreadyWritten) { this.fd = fd; + this.filesFacade = filesFacade; this.mmapAddress = mmapAddress; this.magicWritten = magicAlreadyWritten; } @@ -122,7 +127,7 @@ public void close() { Files.munmap(mmapAddress, FILE_SIZE, MemoryTag.MMAP_DEFAULT); } if (fd >= 0) { - Files.close(fd); + filesFacade.close(fd); } } @@ -136,6 +141,10 @@ public void close() { * stamps the magic and the new FSN atomically. */ public static AckWatermark open(String slotDir) { + return open(FilesFacade.INSTANCE, slotDir); + } + + static AckWatermark open(FilesFacade filesFacade, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; // Decide by size: existing-and-correct -> openRW preserves the // previous session's watermark (defeating which is the whole @@ -143,17 +152,17 @@ public static AckWatermark open(String slotDir) { // wrong-sized -> openCleanRW + allocate creates a fresh // FILE_SIZE-byte file (zero magic, read() reports INVALID until // the first write). - long existing = Files.exists(filePath) ? Files.length(filePath) : -1L; + long existing = filesFacade.exists(filePath) ? filesFacade.length(filePath) : -1L; int fd; if (existing == FILE_SIZE) { - fd = Files.openRW(filePath); + fd = filesFacade.openRW(filePath); } else { - fd = Files.openCleanRW(filePath); - if (fd >= 0 && !Files.allocate(fd, FILE_SIZE)) { + fd = filesFacade.openCleanRW(filePath); + if (fd >= 0 && !filesFacade.allocate(fd, FILE_SIZE)) { // FilesFacade.allocate contract on a false return: // close the fd AND unlink the partial file. - Files.close(fd); - Files.remove(filePath); + filesFacade.close(fd); + filesFacade.remove(filePath); fd = -1; } } @@ -165,7 +174,7 @@ public static AckWatermark open(String slotDir) { long addr = Files.mmap(fd, FILE_SIZE, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { LOG.warn("ack watermark {} could not be mmapped; proceeding without it", filePath); - Files.close(fd); + filesFacade.close(fd); return null; } // Inspect the existing magic once at open time. If it's already @@ -173,7 +182,7 @@ public static AckWatermark open(String slotDir) { // shutdown), the first write() can skip the magic store // entirely and degenerate to a single 8-byte FSN put. int magic = Unsafe.getUnsafe().getInt(addr + MAGIC_OFFSET); - return new AckWatermark(fd, addr, magic == FILE_MAGIC); + return new AckWatermark(filesFacade, fd, addr, magic == FILE_MAGIC); } /** @@ -183,7 +192,11 @@ public static AckWatermark open(String slotDir) { * would only confuse the next session's seed. */ public static void removeOrphan(String slotDir) { - Files.remove(slotDir + "/" + FILE_NAME); + removeOrphan(FilesFacade.INSTANCE, slotDir); + } + + static boolean removeOrphan(FilesFacade filesFacade, String slotDir) { + return filesFacade.remove(slotDir + "/" + FILE_NAME); } /** @@ -203,6 +216,23 @@ public long read() { return Unsafe.getUnsafe().getLong(mmapAddress + FSN_OFFSET); } + /** + * Flushes the mapped bytes and then the backing fd. The caller must sync + * the slot directory after this succeeds so a newly-created watermark's + * directory entry is durable before segment deletion begins. + */ + public void sync() { + if (closed) { + throw new IllegalStateException("ack watermark is closed"); + } + if (filesFacade.msync(mmapAddress, FILE_SIZE, false) != 0) { + throw new IllegalStateException("could not msync ack watermark"); + } + if (filesFacade.fsync(fd) != 0) { + throw new IllegalStateException("could not fsync ack watermark"); + } + } + /** * Atomically updates the persisted FSN. Single 8-byte aligned * store to the mapped region. The first write also stamps the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 01910fcb..1d46c489 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -26,6 +26,7 @@ import io.questdb.client.std.Compat; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; import org.jetbrains.annotations.TestOnly; @@ -92,6 +93,7 @@ public final class CursorSendEngine implements QuietCloseable { // callback allocation failure cannot orphan manager resources. A timed-out // close can then hand it to either manager path without allocating. private final Runnable deferredClose; + private final FilesFacade filesFacade; private final SegmentManager manager; // We own the manager iff the user constructed us with no manager — in that // case close() also stops the manager. When the manager is shared across @@ -269,6 +271,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man this.sfDir = sfDir; this.segmentSizeBytes = segmentSizeBytes; this.manager = manager; + this.filesFacade = manager.filesFacade(); this.ownsManager = ownsManager; this.appendDeadlineNanos = appendDeadlineNanos; @@ -335,7 +338,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // open() returns null on any setup failure so a missing // mmap doesn't take down the engine -- we just fall // back to the bare lowestBase - 1 seed. - watermarkInProgress = AckWatermark.open(sfDir); + watermarkInProgress = AckWatermark.open(filesFacade, sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -386,8 +389,8 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // so the new session's first read() correctly reports // INVALID (magic=0 on a freshly zero-filled file). if (!memoryMode) { - AckWatermark.removeOrphan(sfDir); - watermarkInProgress = AckWatermark.open(sfDir); + AckWatermark.removeOrphan(filesFacade, sfDir); + watermarkInProgress = AckWatermark.open(filesFacade, sfDir); } MmapSegment initial; String initialPath = null; @@ -747,6 +750,7 @@ public synchronized void close() { */ private void finishClose(boolean fullyDrained) { try { + RuntimeException durabilityFailure = null; // On a fully-drained close, persist the final acked FSN through // the still-mapped watermark BEFORE closing the ring/watermark // and BEFORE unlinking any segment file. The manager persists @@ -764,8 +768,14 @@ private void finishClose(boolean fullyDrained) { long finalAckedFsn = ring.ackedFsn(); if (finalAckedFsn >= 0) { watermark.write(finalAckedFsn); + watermark.sync(); + if (filesFacade.fsyncDir(sfDir) != 0) { + throw new IllegalStateException( + "could not fsync SF slot directory before segment cleanup"); + } } - } catch (Throwable ignored) { + } catch (RuntimeException e) { + durabilityFailure = e; } } try { @@ -786,21 +796,22 @@ private void finishClose(boolean fullyDrained) { } catch (Throwable ignored) { } } - if (fullyDrained) { + if (fullyDrained && watermark != null && durabilityFailure == null) { boolean segmentsRemoved = false; try { - segmentsRemoved = unlinkAllSegmentFiles(sfDir); + segmentsRemoved = unlinkAllSegmentFiles(filesFacade, sfDir); } catch (Throwable ignored) { } - // Remove the watermark ONLY once every segment file is - // confirmed gone. The watermark is what keeps residual - // acknowledged segments inert to a successor's recovery; - // removing it while any .sfa file survives would republish - // those already-acknowledged rows. + // Remove the watermark ONLY once every segment unlink is + // durable. Until the directory fsync succeeds, stable storage + // may still contain acknowledged segments and therefore still + // needs the durable watermark. if (segmentsRemoved) { - try { - AckWatermark.removeOrphan(sfDir); - } catch (Throwable ignored) { + if (filesFacade.fsyncDir(sfDir) != 0) { + durabilityFailure = new IllegalStateException( + "could not fsync SF slot directory after segment cleanup"); + } else { + AckWatermark.removeOrphan(filesFacade, sfDir); } } else { LOG.warn("close-time segment cleanup incomplete on slot {}; retaining the ack " @@ -808,6 +819,12 @@ private void finishClose(boolean fullyDrained) { + "engine on this slot recovers them as fully acked and retries the " + "unlink on its own close", sfDir); } + } else if (fullyDrained && watermark == null && sfDir != null) { + LOG.warn("close-time segment cleanup skipped on slot {} because no ack watermark " + + "is available to cover a host crash during unlink", sfDir); + } + if (durabilityFailure != null) { + throw durabilityFailure; } } finally { // Reaching finishClose at all requires established quiescence, so @@ -1247,9 +1264,9 @@ private static long segmentCleanupRank(String name) { * watermark on {@code false} so residual acknowledged segments stay * covered. */ - private static boolean unlinkAllSegmentFiles(String dir) { - if (!io.questdb.client.std.Files.exists(dir)) return true; - long find = io.questdb.client.std.Files.findFirst(dir); + private static boolean unlinkAllSegmentFiles(FilesFacade filesFacade, String dir) { + if (!filesFacade.exists(dir)) return true; + long find = filesFacade.findFirst(dir); if (find < 0) { LOG.warn("close-time unlink could not enumerate {}; " + "any residual sf-*.sfa files will be picked up by the next recovery", dir); @@ -1261,14 +1278,14 @@ private static boolean unlinkAllSegmentFiles(String dir) { try { while (rc > 0) { String name = io.questdb.client.std.Files.utf8ToString( - io.questdb.client.std.Files.findName(find)); - rc = io.questdb.client.std.Files.findNext(find); + filesFacade.findName(find)); + rc = filesFacade.findNext(find); if (name != null && name.endsWith(".sfa")) { names.add(name); } } } finally { - io.questdb.client.std.Files.findClose(find); + filesFacade.findClose(find); } if (rc < 0) { // A partial listing must not drive any unlink: removing only the @@ -1285,7 +1302,7 @@ private static boolean unlinkAllSegmentFiles(String dir) { }); for (int i = 0, n = names.size(); i < n; i++) { String path = dir + "/" + names.get(i); - if (!io.questdb.client.std.Files.remove(path)) { + if (!filesFacade.remove(path)) { LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the " + "residual files stay a contiguous, watermark-covered range", path); return false; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 32ec4b30..b8ae47d9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -244,11 +244,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // terminalError: the only writer runs on the I/O thread under the same // first-writer-wins latch. private volatile QwpDurableAckMismatchException capabilityGapTerminal; - // Failed-stop hand-off flag: set by delegateEngineClose() when an owner's - // close() could not stop the I/O thread and the engine close is therefore - // performed by the I/O thread's exit path. Write-once, owner thread only; - // read by the I/O thread strictly after its shutdown-latch countdown (see - // the handshake contract on delegateEngineClose). + // Failed-stop hand-off callback: set when an owner could not stop the I/O + // thread and must defer worker-reachable cleanup until the thread exits. + // Read strictly after shutdownLatch.countDown(); see delegateClose(). + private volatile Runnable delegatedClose; + // Engine-only hand-off retained for BackgroundDrainer, whose remaining + // resources are owned by its run method rather than by a sender. private volatile boolean engineCloseDelegated; // The latched terminal failure — THE exception every checkError() call // rethrows. Write-once for the loop's lifetime: the only writer is @@ -834,27 +835,20 @@ public synchronized void close() { } /** - * Failed-stop hand-off for the engine. Called by an owner whose - * {@link #close()} threw because the I/O thread would not stop: the owner - * must not free the engine (munmap/Unsafe.free of segment memory) while - * the thread may still touch it with raw {@code Unsafe} reads. Setting - * the delegation flag makes the I/O thread run {@code engine.close()} on - * its exit path, strictly after its last engine access and after the - * shutdown-latch countdown — releasing the slot lock as soon as the - * stuck wire call resolves (bounded by OS timeouts) instead of leaking - * the mapping and lock forever. - *

    - * Returns {@code true} when the I/O thread is still live and has adopted - * the engine close; {@code false} when the thread has already exited — - * the caller must close the engine itself. - *

    - * Memory model — the classic store/load handshake: this method writes the - * volatile flag, then reads the latch count; the exit path counts the - * latch down, then reads the flag. Under the sequential consistency of - * volatile (and AQS latch state) accesses, if this method observes the - * latch still up, the exit path is guaranteed to observe the flag — no - * missed close. If both sides act, {@link CursorSendEngine#close()} is - * synchronized and idempotent, so the double close is benign. + * Hands complete owner cleanup to the I/O thread when it could not be + * stopped. The callback runs after the thread's last client, buffer, and + * engine access. Returns false if the thread already exited, in which case + * the caller must run the callback. The callback itself must be idempotent: + * the latch/callback handshake guarantees execution but permits both sides + * to race after the countdown. + */ + public boolean delegateClose(Runnable closeCallback) { + delegatedClose = closeCallback; + return shutdownLatch.getCount() != 0L; + } + + /** + * Engine-only failed-stop hand-off used by BackgroundDrainer. */ public boolean delegateEngineClose() { engineCloseDelegated = true; @@ -1692,15 +1686,14 @@ private void ioLoop() { } } shutdownLatch.countDown(); - // Failed-stop hand-off (see delegateEngineClose): the owner could - // not free the engine safely while this thread was alive, so the - // engine close — and with it the slot-lock release — happens - // here, strictly after this thread's last engine access. The flag - // is read only after the countDown: the store/load pairing with - // delegateEngineClose's flag-write-then-latch-read guarantees - // either this branch or the owner's fallback runs (or both — - // engine.close() is idempotent). - if (engineCloseDelegated) { + Runnable closeCallback = delegatedClose; + if (closeCallback != null) { + try { + closeCallback.run(); + } catch (Throwable ignored) { + // The owner callback logs individual cleanup failures. + } + } else if (engineCloseDelegated) { try { engine.close(); } catch (Throwable ignored) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 78e5db9f..226e047f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -95,6 +95,10 @@ public final class MmapSegment implements QuietCloseable { // because the consumer must see writes in publication order — once the // producer bumps publishedCursor, every byte before it is fully written. private volatile long publishedCursor; + // Monotonic in-memory link to the segment that immediately follows this + // one. SegmentRing publishes it before promoting the successor to active; + // close deliberately retains it so a cursor can advance after head trim. + private volatile MmapSegment successor; // Bytes between the last valid frame and the file end that look like an // attempted-but-invalid frame write (non-zero bytes at the bail-out // position). Zero for fresh segments and for cleanly partially-filled @@ -438,6 +442,21 @@ public long sizeBytes() { return sizeBytes; } + void linkSuccessor(MmapSegment next) { + if (next == null) { + throw new IllegalArgumentException("successor must not be null"); + } + MmapSegment existing = successor; + if (existing != null && existing != next) { + throw new IllegalStateException("segment successor already linked"); + } + successor = next; + } + + MmapSegment successor() { + return successor; + } + /** * Appends one frame: writes {@code [crc32c | u32 payloadLen | payload]} * starting at the current append cursor, then advances both cursors diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 578c4f07..6bb617ad 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.LockSupport; +import java.util.function.LongSupplier; /** * Background worker that keeps every registered {@link SegmentRing} supplied @@ -51,17 +52,6 @@ * in a JVM). Polls each ring on a configurable tick (default 1 ms) — short * enough that a producer rarely sees {@link SegmentRing#BACKPRESSURE_NO_SPARE} * in the steady state, long enough that an idle JVM doesn't burn CPU. - *

    - * baseSeq race window: the spare is created with - * {@code baseSeq = ring.nextSeqHint()} as observed by the manager. If the - * producer thread appends more frames before the rotation actually fires, - * the spare's baseSeq will be stale and {@link SegmentRing#appendOrFsn} will - * throw on the mismatch check. In practice this is benign — by the time - * {@link SegmentRing#needsHotSpare()} returns true the producer has very - * little room left in the active segment, and the manager polls fast enough - * to install before the producer fills the rest. Hardening to make the race - * impossible (lazy header write at rotation time) is a separate refinement - * deferred to PR2. */ public final class SegmentManager implements QuietCloseable { @@ -77,6 +67,13 @@ public final class SegmentManager implements QuietCloseable { private static final AtomicIntegerFieldUpdater ENTRY_STATE_UPDATER = AtomicIntegerFieldUpdater.newUpdater(RingEntry.class, "state"); private static final Logger LOG = LoggerFactory.getLogger(SegmentManager.class); + private static final int MAX_TRIMS_PER_RING_PASS = 64; + private static final long TRIM_RETRY_INITIAL_NANOS = 4_000_000L; + private static final long TRIM_RETRY_MAX_NANOS = 1_024_000_000L; + private static final int TRIM_RETRY_NONE = 0; + private static final int TRIM_RETRY_POST_BARRIER = 3; + private static final int TRIM_RETRY_PRE_BARRIER = 1; + private static final int TRIM_RETRY_UNLINK = 2; private static final long WORKER_JOIN_TIMEOUT_MILLIS = 5_000L; private final AtomicLong fileGeneration = new AtomicLong(); @@ -97,6 +94,11 @@ public final class SegmentManager implements QuietCloseable { private final ObjList ringSnapshot = new ObjList<>(); private final ObjList rings = new ObjList<>(); private final long segmentSizeBytes; + private final LongSupplier ticks; + // Reused by the worker for one bounded trim quantum. Keeping the unlinked + // prefix here until the post-unlink directory barrier succeeds avoids both + // per-pass allocation and publishing logical removal before it is durable. + private final MmapSegment[] trimBatch = new MmapSegment[MAX_TRIMS_PER_RING_PASS]; // Test seam: runs after a deferred ring-pass cleanup returns. Null in // production; public sender/pool lifecycle tests use it to observe exact // callback completion without sleeps or polling. @@ -126,6 +128,10 @@ public final class SegmentManager implements QuietCloseable { // entry-state check inside the trim block closes for watermark writes and // totalBytes accounting. private volatile Runnable beforeTrimSyncHook; + // Test seam invoked exactly when a retry transition/recovery is logged. + // Null in production; persistent-failure tests use it to prove log bounds + // without binding to a particular SLF4J backend. + private volatile Runnable retryLogHook; // Entry the worker is claiming or servicing, or null between passes. // Volatile publication lets teardown find the entry after deregister() // removes it from rings. RingEntry.state is the authoritative barrier: @@ -174,11 +180,11 @@ public final class SegmentManager implements QuietCloseable { private volatile Thread workerThread; public SegmentManager(long segmentSizeBytes) { - this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE); + this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE, System::nanoTime); } public SegmentManager(long segmentSizeBytes, long pollNanos) { - this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE); + this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE, System::nanoTime); } /** @@ -204,11 +210,22 @@ public SegmentManager(long segmentSizeBytes, long pollNanos) { * hold an initial active plus one hot spare. */ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) { - this(segmentSizeBytes, pollNanos, maxTotalBytes, FilesFacade.INSTANCE); + this(segmentSizeBytes, pollNanos, maxTotalBytes, FilesFacade.INSTANCE, System::nanoTime); } @TestOnly public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes, FilesFacade filesFacade) { + this(segmentSizeBytes, pollNanos, maxTotalBytes, filesFacade, System::nanoTime); + } + + @TestOnly + public SegmentManager( + long segmentSizeBytes, + long pollNanos, + long maxTotalBytes, + FilesFacade filesFacade, + LongSupplier ticks + ) { // The pathScratch field initializer has already allocated its native // buffer by the time this body runs, so a validation throw must free // it or every failed construction leaks 256 bytes of native memory @@ -227,6 +244,11 @@ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes, this.segmentSizeBytes = segmentSizeBytes; this.pollNanos = pollNanos; this.maxTotalBytes = maxTotalBytes; + this.ticks = ticks; + } + + FilesFacade filesFacade() { + return filesFacade; } @Override @@ -586,6 +608,11 @@ public void setBeforeTrimSyncHook(Runnable hook) { this.beforeTrimSyncHook = hook; } + @TestOnly + public void setRetryLogHook(Runnable hook) { + this.retryLogHook = hook; + } + @TestOnly public void setWorkerJoinTimeoutMillis(long millis) { this.workerJoinTimeoutMillis = millis; @@ -691,17 +718,18 @@ private String nextSparePath(String dir) { return displayPath; } - private void serviceRing(RingEntry e) { + private boolean serviceRing(RingEntry e) { // Publish before the CAS so deregister + await can always find the // entry. The state CAS is the ownership decision: if deregister won, // this stale snapshot pass skips the ring without touching it. inService = e; if (!ENTRY_STATE_UPDATER.compareAndSet(e, ENTRY_REGISTERED, ENTRY_IN_SERVICE)) { inService = null; - return; + return false; } + boolean hasMoreTrimmable; try { - serviceRing0(e); + hasMoreTrimmable = serviceRing0(e); } finally { e.finishService(); Runnable cleanup = e.quiescenceCleanup == null @@ -731,9 +759,10 @@ private void serviceRing(RingEntry e) { } } } + return hasMoreTrimmable; } - private void serviceRing0(RingEntry e) { + private boolean serviceRing0(RingEntry e) { // 1. Provision a hot spare if the ring needs one AND we have headroom // under the disk-total cap. Cap check is per-tick; if we're capped // here, the ring stays in BACKPRESSURE_NO_SPARE until trim (step 2) @@ -829,62 +858,207 @@ private void serviceRing0(RingEntry e) { } } - // 2. Trim any segments that the ring says are fully acked. For - // memory-mode rings, "trim" is just close() (Unsafe.free) — no - // file to unlink. - // - // The watermark write and totalBytes commit are registration-gated - // under `lock` so stale worker snapshots cannot touch the - // engine-owned watermark or mutate accounting after deregister() - // returns. Trimming still runs for stale snapshots: a successful - // close + unlink transfers the fully-acked segment out of the ring, - // preserving cleanup without touching deregistered accounting. - // - // munmap + unlink stay outside the lock — they can be slow - // and shouldn't block register/deregister or sibling rings. + // 2. Trim any segments that the ring says are fully acked. Memory + // mode only frees native memory. Disk mode first makes the current + // cumulative ACK durable, then durably establishes the watermark's + // directory entry, unlinks one bounded batch, and finally commits + // those directory removals before ring/accounting state changes. + // No syscall runs under the manager lock. Runnable hook = beforeTrimSyncHook; if (hook != null) { hook.run(); } + MmapSegment first = e.ring.firstTrimmable(); + if (first == null) { + // Preserve the cheap mmap-only watermark cadence for ACKs that do + // not yet cover a complete sealed segment. + synchronized (lock) { + if (e.isRegistered() && e.watermark != null) { + long currentAck = e.ring.ackedFsn(); + if (currentAck > e.lastPersistedAck) { + e.watermark.write(currentAck); + e.lastPersistedAck = currentAck; + } + } + } + return false; + } + + if (memoryMode) { + int trimmed = 0; + while (trimmed < MAX_TRIMS_PER_RING_PASS) { + MmapSegment segment = e.ring.firstTrimmable(); + if (segment == null) { + break; + } + try { + segment.close(); + synchronized (lock) { + if (e.ring.removeTrimmable(segment)) { + trimmed++; + if (e.isRegistered()) { + totalBytes -= segment.sizeBytes(); + } + } + } + } catch (Throwable t) { + LOG.warn("Failed to trim memory segment", t); + return false; + } + } + return trimmed == MAX_TRIMS_PER_RING_PASS && e.ring.firstTrimmable() != null; + } + + // A deferred disk retry does no sync, unlink, or logging work. Signed + // subtraction is the standard wrap-safe deadline comparison because + // the bounded delay is many orders of magnitude below half the long + // range, even when the monotonic clock wraps. + long now = ticks.getAsLong(); + if (e.trimRetryDelayNanos != 0 && now - e.trimRetryAtNanos < 0) { + return false; + } + + // A disk segment cannot be safely unlinked without durable ACK cover. + // The registration check and mmap store are atomic with deregister. + // Once the store wins, deregistration may proceed, but its owner must + // await this in-service pass before closing the watermark or ring. + if (e.watermark == null) { + if (!e.missingWatermarkLogged) { + e.missingWatermarkLogged = true; + LOG.warn("Cannot durably trim acknowledged segments in {} without an ack watermark", e.dir); + } + return false; + } + long durableAck; + synchronized (lock) { + if (!e.isRegistered()) { + return false; + } + durableAck = e.ring.ackedFsn(); + if (durableAck > e.lastPersistedAck) { + e.watermark.write(durableAck); + e.lastPersistedAck = durableAck; + } + } + try { + e.watermark.sync(); + if (filesFacade.fsyncDir(e.dir) != 0) { + recordTrimFailure(e, TRIM_RETRY_PRE_BARRIER, now, null); + return false; + } + } catch (Throwable t) { + recordTrimFailure(e, TRIM_RETRY_PRE_BARRIER, now, t); + return false; + } + boolean trimFailed = false; - while (true) { - MmapSegment segment = e.ring.firstTrimmable(); - if (segment == null) { + Throwable trimFailure = null; + int unlinked = 0; + MmapSegment segment = first; + long coveredAck = durableAck; + while (segment != null && unlinked < MAX_TRIMS_PER_RING_PASS) { + long lastSeq = segment.baseSeq() + segment.frameCount() - 1L; + if (lastSeq > coveredAck) { break; } + MmapSegment next = e.ring.nextSealedAfter(segment); String path = segment.path(); try { segment.close(); - if (path != null && !filesFacade.remove(path)) { + // A retry after a post-unlink directory-sync failure sees an + // already absent path. Treat absence as the prior successful + // unlink and retry the batch barrier rather than wedging. + if (!filesFacade.remove(path) && filesFacade.exists(path)) { trimFailed = true; - LOG.warn("Failed to unlink trimmed segment {}", path); break; } - synchronized (lock) { - if (e.ring.removeTrimmable(segment) && e.isRegistered()) { - totalBytes -= segment.sizeBytes(); - } - } + trimBatch[unlinked++] = segment; + segment = next; } catch (Throwable t) { trimFailed = true; - LOG.warn("Failed to trim segment {}", path == null ? "" : path, t); + trimFailure = t; break; } } - // An unlink failure leaves the acknowledged segment in the ring and - // on disk. Do not advance the persisted watermark past it: recovery - // must continue to see the failed path as live bookkeeping rather - // than infer that trim committed. A later successful retry persists - // the current cumulative ACK normally. - synchronized (lock) { - if (!trimFailed && e.isRegistered() && e.watermark != null) { - long currentAck = e.ring.ackedFsn(); - if (currentAck > e.lastPersistedAck) { - e.watermark.write(currentAck); - e.lastPersistedAck = currentAck; + + if (unlinked > 0) { + try { + if (filesFacade.fsyncDir(e.dir) != 0) { + for (int i = 0; i < unlinked; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, null); + return false; + } + } catch (Throwable t) { + for (int i = 0; i < unlinked; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, t); + return false; + } + synchronized (lock) { + for (int i = 0; i < unlinked; i++) { + MmapSegment removed = trimBatch[i]; + trimBatch[i] = null; + if (e.ring.removeTrimmable(removed) && e.isRegistered()) { + totalBytes -= removed.sizeBytes(); + } } } } + if (trimFailed) { + recordTrimFailure(e, TRIM_RETRY_UNLINK, now, trimFailure); + return false; + } + recordTrimSuccess(e); + return unlinked == MAX_TRIMS_PER_RING_PASS && e.ring.firstTrimmable() != null; + } + + private void recordTrimFailure(RingEntry e, int failureKind, long now, Throwable failure) { + long delay = e.trimRetryDelayNanos == 0 + ? TRIM_RETRY_INITIAL_NANOS + : Math.min(e.trimRetryDelayNanos << 1, TRIM_RETRY_MAX_NANOS); + e.trimRetryAtNanos = now + delay; + e.trimRetryDelayNanos = delay; + if (e.trimRetryFailureKind != failureKind) { + e.trimRetryFailureKind = failureKind; + if (failure == null) { + LOG.warn("Durable segment trim failed in {} during {} (retry delayed)", + e.dir, trimFailureName(failureKind)); + } else { + LOG.warn("Durable segment trim failed in {} during {} (retry delayed)", + e.dir, trimFailureName(failureKind), failure); + } + Runnable hook = retryLogHook; + if (hook != null) { + hook.run(); + } + } + } + + private void recordTrimSuccess(RingEntry e) { + if (e.trimRetryDelayNanos != 0) { + e.trimRetryAtNanos = 0; + e.trimRetryDelayNanos = 0; + e.trimRetryFailureKind = TRIM_RETRY_NONE; + LOG.info("Durable segment trim recovered in {}", e.dir); + Runnable hook = retryLogHook; + if (hook != null) { + hook.run(); + } + } + } + + private static String trimFailureName(int failureKind) { + switch (failureKind) { + case TRIM_RETRY_PRE_BARRIER: + return "covering barrier"; + case TRIM_RETRY_UNLINK: + return "segment unlink"; + default: + return "directory commit barrier"; + } } private void workerLoop() { @@ -899,16 +1073,21 @@ private void workerLoop() { ringSnapshot.add(rings.getQuick(i)); } } + boolean hasMoreTrimmable = false; for (int i = 0, n = ringSnapshot.size(); i < n; i++) { if (!running) break; - serviceRing(ringSnapshot.getQuick(i)); + if (serviceRing(ringSnapshot.getQuick(i))) { + hasMoreTrimmable = true; + } } // Drop strong refs so a deregistered ring becomes collectable // before the next tick (otherwise the snapshot pins it for up // to pollNanos after deregister). ringSnapshot.clear(); if (!running) break; - LockSupport.parkNanos(pollNanos); + if (!hasMoreTrimmable) { + LockSupport.parkNanos(pollNanos); + } } } finally { // If a timed-out close() abandoned the reap, it handed @@ -969,6 +1148,14 @@ private static final class RingEntry { // Survives across multiple serviceRing ticks and avoids a // write-storm when ackedFsn is steady. long lastPersistedAck = -1L; + // Prevents a legacy disk registration without a watermark from + // flooding the log on every manager tick. + boolean missingWatermarkLogged; + // Zero-allocation manager-thread-only retry state. The deadline uses + // the manager's monotonic clock and the delay doubles to a fixed cap. + long trimRetryAtNanos; + long trimRetryDelayNanos; + int trimRetryFailureKind; // Updated lock-free by deferUntilRingQuiescent and pass completion. // The field updater avoids one AtomicReference allocation per ring. volatile Runnable quiescenceCleanup; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 8d929ae1..8f5fddbb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -63,6 +63,9 @@ public final class SegmentRing implements QuietCloseable { /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */ public static final long PAYLOAD_TOO_LARGE = -2L; private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class); + // Tally of sealed-list entries inspected by nextSealedAfter(). Test-only + // operation count for deterministic traversal-complexity assertions. + private static long nextSealedComparisons; // Tally of baseSeq comparisons performed by sortByBaseSeq across every // openExisting() recovery on this JVM. Used by SegmentRingTest to // assert the sort stays O(N log N) without relying on wall-clock time @@ -70,6 +73,9 @@ public final class SegmentRing implements QuietCloseable { // in production: one volatile-free add per partition pass, dwarfed by // the mmap I/O the recovery does on every segment. private static long sortComparisons; + // References copied while compacting the logical sealed-segment head. + // Test-only operation count for deterministic trim-complexity assertions. + private static long trimMovedReferences; private final long maxBytesPerSegment; // Sealed segments in baseSeq order, oldest first. Active is held separately. // Single-writer (producer thread, on rotation); single-reader at trim time @@ -78,6 +84,9 @@ public final class SegmentRing implements QuietCloseable { // looks at sealedSegments after observing a higher ackedFsn, by which // point the producer thread's add to sealedSegments has retired. private final ObjList sealedSegments = new ObjList<>(); + // Logical head of sealedSegments. Head removal nulls one entry and advances + // this index; occasional compaction bounds unused prefix slots. + private int sealedHead; // High-water byte offset within the active segment at which we proactively // ask the segment manager to provision a spare (if one isn't already // installed). Computed once as 3/4 of segment capacity -- leaves the manager @@ -286,6 +295,11 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) { + " check disk health"); } } + // Publish the immutable in-memory successor chain before exposing + // the recovered ring. The final sealed segment links to the active. + for (int i = 1, n = opened.size(); i < n; i++) { + opened.get(i - 1).linkSuccessor(opened.get(i)); + } // The newest segment becomes the active. Even if it's full, that's OK: // the next appendOrFsn returns BACKPRESSURE_NO_SPARE, the manager // installs a hot spare, the producer rotates. Same fast path as a @@ -379,15 +393,18 @@ public long appendOrFsn(long payloadAddr, int payloadLen) { // full, so its frameCount is stable, and (b) the spare hasn't been // appended to yet (rebaseSeq enforces that). The segment manager's // earlier guess at baseSeq is irrelevant. - long actualBase = active.baseSeq() + active.frameCount(); + MmapSegment previous = active; + long actualBase = previous.baseSeq() + previous.frameCount(); spare.rebaseSeq(actualBase); - // Mutate sealedSegments under the same monitor used by - // snapshotSealedSegments -- the I/O thread reads through that - // path and must not see a half-resized ObjList. + // Publish the successor before the volatile active promotion. The + // same monitor protects the sealed list and nextSealedAfter's trim + // fallback, while the volatile link also remains readable from a + // current segment after the manager removes and closes it. synchronized (this) { - sealedSegments.add(active); + previous.linkSuccessor(spare); + sealedSegments.add(previous); + active = spare; } - active = spare; hotSpare = null; // Fresh active just consumed the spare → ask the manager to start // making the next one immediately, before this segment fills. @@ -439,13 +456,11 @@ public synchronized void close() { hotSpare.close(); hotSpare = null; } - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s != null) { - s.close(); - } + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + sealedSegments.get(i).close(); } sealedSegments.clear(); + sealedHead = 0; } /** @@ -461,10 +476,8 @@ public synchronized ObjList drainTrimmable() { ObjList out = null; // Sealed segments are in baseSeq order, oldest first; once we hit one // that isn't fully acked, none of the later ones can be either. - // Synchronized so the I/O thread's snapshotSealedSegments() can't - // race against the remove(0) shuffling slots underneath it. - while (sealedSegments.size() > 0) { - MmapSegment s = sealedSegments.get(0); + while (sealedHead < sealedSegments.size()) { + MmapSegment s = sealedSegments.get(sealedHead); long lastSeq = s.baseSeq() + s.frameCount() - 1; if (lastSeq > acked) { break; @@ -473,7 +486,7 @@ public synchronized ObjList drainTrimmable() { out = new ObjList<>(); } out.add(s); - sealedSegments.remove(0); + removeSealedHead(); } return out; } @@ -490,7 +503,7 @@ public synchronized ObjList drainTrimmable() { * scan cost doesn't matter. */ public synchronized MmapSegment findSegmentContaining(long fsn) { - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { MmapSegment s = sealedSegments.get(i); long base = s.baseSeq(); if (fsn >= base && fsn < base + s.frameCount()) { @@ -513,7 +526,7 @@ public synchronized MmapSegment findSegmentContaining(long fsn) { * fallback -- see {@link #nextSealedAfter(MmapSegment)}. */ public synchronized MmapSegment firstSealed() { - return sealedSegments.size() > 0 ? sealedSegments.get(0) : null; + return sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null; } /** @@ -523,10 +536,10 @@ public synchronized MmapSegment firstSealed() { * bookkeeping or allow its identifier to be reused. */ public synchronized MmapSegment firstTrimmable() { - if (sealedSegments.size() == 0) { + if (sealedHead == sealedSegments.size()) { return null; } - MmapSegment segment = sealedSegments.get(0); + MmapSegment segment = sealedSegments.get(sealedHead); long lastSeq = segment.baseSeq() + segment.frameCount() - 1; return lastSeq <= ackedFsn ? segment : null; } @@ -551,7 +564,7 @@ public synchronized MmapSegment firstTrimmable() { */ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) { long best = -1L; - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { long fsn = sealedSegments.get(i).findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen); if (fsn > best) { best = fsn; @@ -571,7 +584,8 @@ public MmapSegment getActive() { * concurrent rotation. Cross-thread readers (typically the I/O loop) * should use {@link #snapshotSealedSegments(MmapSegment[])} instead. */ - public ObjList getSealedSegments() { + public synchronized ObjList getSealedSegments() { + compactSealedSegments(); return sealedSegments; } @@ -615,21 +629,28 @@ public boolean needsHotSpare() { * outpaces the I/O thread and sealed segments accumulate well beyond * any reasonable snapshot-array size. *

    - * Identity match is intentionally avoided: we compare {@code baseSeq} - * so the loop is robust against the case where {@code current} was - * trimmed out from under us (already ACK'd before the I/O thread - * advanced) -- we still return the next segment in baseSeq order rather - * than failing. Synchronized against rotation. + * Each segment publishes its successor once, before rotation exposes that + * successor as active. A constant-time head check detects when trimming + * removed the immediate successor and falls forward to the oldest live + * sealed segment. Synchronized against rotation and head removal. */ public synchronized MmapSegment nextSealedAfter(MmapSegment current) { - long currentBase = current.baseSeq(); - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s.baseSeq() > currentBase) { - return s; - } + nextSealedComparisons++; + MmapSegment successor = current.successor(); + if (successor == null) { + return null; } - return null; + if (successor == active) { + return null; + } + MmapSegment first = sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null; + if (first != null && successor.baseSeq() >= first.baseSeq()) { + return successor; + } + // Head trimming may have removed the immediate successor while the + // I/O cursor still held an older segment. Trims only remove a prefix, + // so the current head is the first live segment after that prefix. + return first; } /** @@ -654,14 +675,14 @@ public long publishedFsn() { * Returns false if concurrent lifecycle activity changed the head. */ public synchronized boolean removeTrimmable(MmapSegment segment) { - if (sealedSegments.size() == 0 || sealedSegments.get(0) != segment) { + if (sealedHead == sealedSegments.size() || sealedSegments.get(sealedHead) != segment) { return false; } long lastSeq = segment.baseSeq() + segment.frameCount() - 1; if (lastSeq > ackedFsn) { return false; } - sealedSegments.remove(0); + removeSealedHead(); return true; } @@ -694,17 +715,12 @@ public void setManagerWakeup(Runnable wakeup) { * the I/O loop is about to do. */ public synchronized int snapshotSealedSegments(MmapSegment[] target) { - int n = sealedSegments.size(); - if (n > target.length) { - for (int i = 0; i < target.length; i++) { - target[i] = sealedSegments.get(i); - } - return -1; + int n = sealedSegments.size() - sealedHead; + int copyCount = Math.min(n, target.length); + for (int i = 0; i < copyCount; i++) { + target[i] = sealedSegments.get(sealedHead + i); } - for (int i = 0; i < n; i++) { - target[i] = sealedSegments.get(i); - } - return n; + return n > target.length ? -1 : n; } /** @@ -720,13 +736,38 @@ public synchronized long totalSegmentBytes() { if (a != null) total += a.sizeBytes(); MmapSegment hs = hotSpare; if (hs != null) total += hs.sizeBytes(); - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s != null) total += s.sizeBytes(); + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + total += sealedSegments.get(i).sizeBytes(); } return total; } + private void compactSealedSegments() { + if (sealedHead > 0) { + int liveCount = sealedSegments.size() - sealedHead; + trimMovedReferences += liveCount; + sealedSegments.remove(0, sealedHead - 1); + sealedHead = 0; + } + } + + private void removeSealedHead() { + sealedSegments.setQuick(sealedHead++, null); + int size = sealedSegments.size(); + if (sealedHead == size) { + sealedSegments.clear(); + sealedHead = 0; + } else if (sealedHead >= 64 && sealedHead >= size - sealedHead) { + compactSealedSegments(); + } + } + + /** Returns the sealed-list operation count used by traversal tests. */ + @TestOnly + public static long getNextSealedComparisons() { + return nextSealedComparisons; + } + /** * Returns the cumulative count of baseSeq comparisons performed by * {@link #sortByBaseSeq} since the last {@link #resetSortComparisons()} @@ -742,12 +783,30 @@ public static long getSortComparisons() { return sortComparisons; } + /** Returns the references moved by sealed-list compaction. */ + @TestOnly + public static long getTrimMovedReferences() { + return trimMovedReferences; + } + + /** Zeroes the counter exposed via {@link #getNextSealedComparisons()}. */ + @TestOnly + public static void resetNextSealedComparisons() { + nextSealedComparisons = 0; + } + /** Zeroes the counter exposed via {@link #getSortComparisons()}. */ @TestOnly public static void resetSortComparisons() { sortComparisons = 0; } + /** Zeroes the counter exposed via {@link #getTrimMovedReferences()}. */ + @TestOnly + public static void resetTrimMovedReferences() { + trimMovedReferences = 0; + } + /** * In-place quicksort over {@code list[lo, hi)} keyed by ascending * {@code baseSeq}. Median-of-three pivot avoids the pathological O(N²) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java index 2dec7668..541f7dcf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java @@ -27,6 +27,7 @@ import io.questdb.client.SenderConnectionEvent; import io.questdb.client.SenderConnectionListener; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -146,6 +147,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * Non-blocking enqueue. Always admits the new event unless the dispatcher * is closed or {@code event} is null. Returns {@code true} when the new diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java index aaa0e4f9..fb796712 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java @@ -27,6 +27,7 @@ import io.questdb.client.SenderError; import io.questdb.client.SenderErrorHandler; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -209,6 +210,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * True if at least one error has been delivered to a user-installed * (non-default) handler since this dispatcher started. Used by diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java index 2c0e90a1..9e01a630 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java @@ -26,6 +26,7 @@ import io.questdb.client.SenderProgressHandler; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -172,6 +173,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * Replace the user-supplied handler. Effective immediately for any * subsequent delivery. Pass {@code null} to install the no-op default. diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index e401f864..1bd60f3f 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -119,6 +119,10 @@ public final class SenderPool implements AutoCloseable { // RuntimeException Throwable (e.g. an -ea AssertionError) mid-prewarm, // exercising the Error-safe delegate cleanup loop. private final IntFunction senderFactory; + // Test seam: runs immediately after a delegate factory returns, before + // listener registration and SenderSlot construction. Null in production; + // error-safety tests inject a preallocated throwable at this ownership gap. + private final Runnable postFactoryHook; // Factory for startup-recovery delegates. Distinct from senderFactory so a // recoverer can force a non-blocking initial connect (initial_connect_mode= // OFF) regardless of user config: a recovery build runs on the @@ -189,6 +193,9 @@ public final class SenderPool implements AutoCloseable { // retiredSlots and returns any index whose flock has since dropped. // Guarded by lock; only ever ticks for SF slots. private int leakedSlots; + // Deterministic white-box complexity counter. Counts delegate release + // probes performed by direct callbacks and fallback scans. Guarded by lock. + private long retiredSlotProbeCount; // The retired slots behind the leakedSlots count: runtime reclaim paths // (discardBroken/reapIdle via reclaimSlot) and the in-range startup- // recovery pass (recoverOneSlotStep, which retains the recoverer slot for @@ -284,6 +291,25 @@ public SenderPool( deferStartupRecovery, null, null, null); } + // Test-only constructor adding a deterministic fault hook for the ownership + // gap after a delegate factory returns. + @TestOnly + public SenderPool( + String configurationString, + int minSize, + int maxSize, + long acquireTimeoutMillis, + long idleTimeoutMillis, + long maxLifetimeMillis, + IntFunction senderFactory, + boolean deferStartupRecovery, + Runnable postFactoryHook + ) { + this(configurationString, minSize, maxSize, acquireTimeoutMillis, + idleTimeoutMillis, maxLifetimeMillis, senderFactory, + deferStartupRecovery, null, null, null, postFactoryHook); + } + // Full constructor adding the user-supplied ingest callbacks (error // handler, connection listener and background-drainer listener), applied // to every Sender the pool builds (see buildManagedSlotSender). The public @@ -301,6 +327,26 @@ public SenderPool( SenderErrorHandler errorHandler, SenderConnectionListener connectionListener, BackgroundDrainerListener drainerListener + ) { + this(configurationString, minSize, maxSize, acquireTimeoutMillis, + idleTimeoutMillis, maxLifetimeMillis, senderFactory, + deferStartupRecovery, errorHandler, connectionListener, + drainerListener, null); + } + + private SenderPool( + String configurationString, + int minSize, + int maxSize, + long acquireTimeoutMillis, + long idleTimeoutMillis, + long maxLifetimeMillis, + IntFunction senderFactory, + boolean deferStartupRecovery, + SenderErrorHandler errorHandler, + SenderConnectionListener connectionListener, + BackgroundDrainerListener drainerListener, + Runnable postFactoryHook ) { if (minSize < 0 || maxSize < 1 || minSize > maxSize) { throw new IllegalArgumentException("invalid pool sizing: min=" + minSize + ", max=" + maxSize); @@ -319,6 +365,7 @@ public SenderPool( this.acquireTimeoutMillis = acquireTimeoutMillis; this.idleTimeoutMillis = idleTimeoutMillis; this.maxLifetimeMillis = maxLifetimeMillis; + this.postFactoryHook = postFactoryHook; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); this.retiredSlots = new ArrayList<>(maxSize); @@ -564,7 +611,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { // the retirement would be permanent even after the release // (fatal at maxSize=1: every later borrow would time out). leakedSlots++; - retiredSlots.add(retained[0]); + addRetiredSlot(retained[0]); LOG.warn("startup SF recovery: slot {} retired: delegate close() returned with " + "the flock still held (I/O or manager worker did not stop); pool capacity reduced by 1, " + "now {} of {} usable [leakedSlots={}]; the slot is re-probed and recovered " @@ -699,8 +746,9 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, stopScan = true; } } catch (Throwable drainErr) { - LOG.warn("startup SF recovery: drain failed for slot {} ({})", + LOG.warn("startup SF recovery: drain failed for slot {} ({}); deferring it", slotPath, drainErr.toString()); + stopScan = true; } finally { try { recoverer.delegate().close(); @@ -710,8 +758,9 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, } } } catch (Throwable scanErr) { - LOG.warn("startup SF recovery: scan failed for slot {} ({})", + LOG.warn("startup SF recovery: scan failed for slot {} ({}); deferring it", slotPath, scanErr.toString()); + stopScan = true; } if (recoverer != null && !flockReleased(recoverer)) { retainedOut[0] = recoverer; @@ -1212,12 +1261,39 @@ public int leakedSlotCount() { } } - private SenderSlot createUnlocked(int slotIndex) { - Sender delegate = senderFactory.apply(slotIndex); - if (delegate instanceof QwpWebSocketSender) { - ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(this::recoverReleasedSlots); + private SenderSlot createSlot(IntFunction factory, int slotIndex) { + Sender delegate = factory.apply(slotIndex); + try { + if (postFactoryHook != null) { + postFactoryHook.run(); + } + SenderSlot slot = new SenderSlot(delegate, this, slotIndex); + if (delegate instanceof QwpWebSocketSender) { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener( + () -> recoverReleasedSlot(slot)); + } + return slot; + } catch (Throwable failure) { + if (delegate instanceof QwpWebSocketSender) { + try { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(null); + } catch (Throwable deregistrationFailure) { + addSuppressed(failure, deregistrationFailure); + } + } + if (delegate != null) { + try { + delegate.close(); + } catch (Throwable closeFailure) { + addSuppressed(failure, closeFailure); + } + } + throw failure; } - return new SenderSlot(delegate, this, slotIndex); + } + + private SenderSlot createUnlocked(int slotIndex) { + return createSlot(senderFactory, slotIndex); } /** @@ -1228,11 +1304,18 @@ private SenderSlot createUnlocked(int slotIndex) { * {@link #drainCandidateSlotForRecovery}. */ private SenderSlot createRecoverer(int slotIndex) { - Sender delegate = recoverySenderFactory.apply(slotIndex); - if (delegate instanceof QwpWebSocketSender) { - ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(this::recoverReleasedSlots); + return createSlot(recoverySenderFactory, slotIndex); + } + + private static void addSuppressed(Throwable failure, Throwable cleanupFailure) { + if (failure != cleanupFailure) { + try { + failure.addSuppressed(cleanupFailure); + } catch (Throwable ignored) { + // Preserve the original construction failure even if recording + // the secondary cleanup failure cannot allocate. + } } - return new SenderSlot(delegate, this, slotIndex); } private Sender defaultSender(int slotIndex) { @@ -1404,7 +1487,7 @@ private boolean reclaimSlot(SenderSlot s, String context) { return true; } leakedSlots++; - retiredSlots.add(s); + addRetiredSlot(s); LOG.warn("SF slot {} retired{}: delegate close() returned with the flock still held " + "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable " + "[leakedSlots={}]; the slot is re-probed and recovered if the worker releases the flock later", @@ -1412,10 +1495,25 @@ private boolean reclaimSlot(SenderSlot s, String context) { return false; } - private void recoverReleasedSlots() { + private boolean recoverReleasedSlot(SenderSlot s) { lock.lock(); try { - reprobeRetiredSlots(); + int retiredIndex = s.retiredIndex(); + if (retiredIndex < 0 + || retiredIndex >= retiredSlots.size() + || retiredSlots.get(retiredIndex) != s) { + // The callback raced retirement, or is stale/duplicate. The + // retirement path probes before insertion, and periodic scans + // remain as a fallback, so no capacity can be lost here. + return false; + } + retiredSlotProbeCount++; + if (!flockReleased(s)) { + return false; + } + recoverRetiredSlotAt(retiredIndex); + slotReleased.signalAll(); + return true; } finally { lock.unlock(); } @@ -1439,22 +1537,10 @@ private boolean reprobeRetiredSlots() { boolean recovered = false; for (int i = retiredSlots.size() - 1; i >= 0; i--) { SenderSlot s = retiredSlots.get(i); + retiredSlotProbeCount++; if (flockReleased(s)) { - // Order is irrelevant. Swap with the tail before removing so - // each recovery does O(1) list work instead of shifting the - // remaining retired entries. The tail has already been probed - // by this reverse scan and, if still present, is unreleased. - int last = retiredSlots.size() - 1; - if (i < last) { - retiredSlots.set(i, retiredSlots.get(last)); - } - retiredSlots.remove(last); - leakedSlots--; - freeSlotIndex(s.slotIndex()); + recoverRetiredSlotAt(i); recovered = true; - LOG.info("SF slot {} recovered: deferred cleanup released the flock after retirement; " + - "pool capacity restored, now {} of {} usable [leakedSlots={}]", - s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); } } if (recovered) { @@ -1462,4 +1548,26 @@ private boolean reprobeRetiredSlots() { } return recovered; } + + private void addRetiredSlot(SenderSlot s) { + s.retiredIndex(retiredSlots.size()); + retiredSlots.add(s); + } + + private void recoverRetiredSlotAt(int retiredIndex) { + SenderSlot s = retiredSlots.get(retiredIndex); + int last = retiredSlots.size() - 1; + if (retiredIndex < last) { + SenderSlot moved = retiredSlots.get(last); + retiredSlots.set(retiredIndex, moved); + moved.retiredIndex(retiredIndex); + } + retiredSlots.remove(last); + s.retiredIndex(-1); + leakedSlots--; + freeSlotIndex(s.slotIndex()); + LOG.info("SF slot {} recovered: deferred cleanup released the flock after retirement; " + + "pool capacity restored, now {} of {} usable [leakedSlots={}]", + s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); + } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderSlot.java b/core/src/main/java/io/questdb/client/impl/SenderSlot.java index 19c93671..5d9ed6ff 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderSlot.java +++ b/core/src/main/java/io/questdb/client/impl/SenderSlot.java @@ -48,6 +48,11 @@ final class SenderSlot { private final Sender delegate; private final SenderPool pool; private final int slotIndex; + // Index in SenderPool.retiredSlots, or -1 while this slot is not retired. + // Guarded by the pool lock. The pool maintains this index across swap + // removals so the delegate's release callback can recover this exact slot + // without scanning every other retired slot. + private int retiredIndex = -1; // Monotonic lease id. Mutated only under the SenderPool lock (bumped in // borrow() when the slot is handed out and in giveBack()/discardBroken() // when it is returned). A PooledSender wrapper captures it live for its @@ -112,6 +117,14 @@ SenderPool pool() { return pool; } + int retiredIndex() { + return retiredIndex; + } + + void retiredIndex(int retiredIndex) { + this.retiredIndex = retiredIndex; + } + int slotIndex() { return slotIndex; } diff --git a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java index dca93e84..248ad61e 100644 --- a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java @@ -86,6 +86,11 @@ public int fsync(int fd) { return Files.fsync(fd); } + @Override + public int fsyncDir(String dir) { + return Files.fsyncDir(dir); + } + @Override public long length(int fd) { return Files.length(fd); @@ -106,6 +111,11 @@ public int mkdir(String path, int mode) { return Files.mkdir(path, mode); } + @Override + public int msync(long addr, long len, boolean async) { + return Files.msync(addr, len, async); + } + @Override public int openCleanRW(String path) { return Files.openCleanRW(path); diff --git a/core/src/main/java/io/questdb/client/std/Files.java b/core/src/main/java/io/questdb/client/std/Files.java index 02756c37..7a4038ed 100644 --- a/core/src/main/java/io/questdb/client/std/Files.java +++ b/core/src/main/java/io/questdb/client/std/Files.java @@ -408,6 +408,20 @@ public static String utf8ToString(long nameZ) { */ public static native int fsync(int fd); + /** + * Forces directory-entry updates under {@code dir} to durable storage. + * Returns 0 on success and non-zero on failure. Callers use this after + * unlinking files whose absence must survive a host crash. + */ + public static int fsyncDir(String dir) { + long ptr = pathPtr(dir); + try { + return fsyncDir0(ptr); + } finally { + freePathPtr(ptr); + } + } + /** * Truncates the file to exactly {@code size} bytes via {@code ftruncate}. * Returns {@code true} on success. Does NOT reserve disk space — the @@ -555,6 +569,8 @@ public static void munmap(long address, long len, int memoryTag) { static native int close0(int fd); + static native int fsyncDir0(long lpszName); + static native int openRO0(long lpszName); static native int openRW0(long lpszName); diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java index 1b408cf4..60000a3b 100644 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java @@ -85,6 +85,10 @@ public interface FilesFacade { int fsync(int fd); + default int fsyncDir(String dir) { + return Files.fsyncDir(dir); + } + long length(int fd); /** @@ -107,6 +111,10 @@ public interface FilesFacade { int mkdir(String path, int mode); + default int msync(long addr, long len, boolean async) { + return Files.msync(addr, len, async); + } + int openCleanRW(String path); /** diff --git a/core/src/main/java/io/questdb/client/std/ObjList.java b/core/src/main/java/io/questdb/client/std/ObjList.java index 9e0e7b29..34d989ae 100644 --- a/core/src/main/java/io/questdb/client/std/ObjList.java +++ b/core/src/main/java/io/questdb/client/std/ObjList.java @@ -149,7 +149,7 @@ public void remove(int from, int to) { System.arraycopy(buffer, to + 1, buffer, from, move); } pos = Math.max(0, pos - (to - from + 1)); - Arrays.fill(buffer, pos, buffer.length - 1, null); + Arrays.fill(buffer, pos, buffer.length, null); } public void remove(int index) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index f2c560ea..63553ebc 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -26,14 +26,19 @@ import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.Sender; +import io.questdb.client.SenderConnectionEvent; +import io.questdb.client.SenderError; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; -import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; @@ -457,6 +462,117 @@ public void testDelegatedIoThreadEngineCloseFlipsSlotLockReleased() throws Excep }); } + @Test(timeout = 30_000L) + public void testFailedIoStopReclaimsSenderResourcesAfterWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-full-cleanup-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + CountDownLatch enteredConnect = new CountDownLatch(1); + CountDownLatch releaseConnect = new CountDownLatch(1); + AtomicReference ioThreadRef = new AtomicReference<>(); + StubWebSocketClient loopClient = new StubWebSocketClient(); + StubWebSocketClient senderClient = new StubWebSocketClient(); + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + CursorWebSocketSendLoop loop = null; + try { + CursorWebSocketSendLoop.ReconnectFactory stuckConnect = () -> { + ioThreadRef.set(Thread.currentThread()); + enteredConnect.countDown(); + releaseConnect.await(); + return loopClient; + }; + loop = new CursorWebSocketSendLoop( + null, engine, 0L, 1_000L, stuckConnect, + 5_000L, 100L, 5_000L, false); + loop.start(); + Assert.assertTrue("I/O thread never reached the connect factory", + enteredConnect.await(5, TimeUnit.SECONDS)); + + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setConnectionListener(event -> { + }); + sender.setErrorHandler(error -> { + }); + sender.setProgressHandler(ackedFsn -> { + }); + sender.setClientForTesting(senderClient); + sender.setCursorEngine(engine, true); + sender.setCursorSendLoopForTesting(loop); + + SenderConnectionDispatcher connectionDispatcher = sender.getConnectionDispatcherForTesting(); + SenderErrorDispatcher errorDispatcher = sender.getErrorDispatcherForTesting(); + SenderProgressDispatcher progressDispatcher = sender.getProgressDispatcherForTesting(); + Assert.assertTrue(connectionDispatcher.offer(new SenderConnectionEvent( + SenderConnectionEvent.Kind.DISCONNECTED, + null, SenderConnectionEvent.NO_PORT, + null, SenderConnectionEvent.NO_PORT, + SenderConnectionEvent.NO_ATTEMPT_NUMBER, + SenderConnectionEvent.NO_ROUND_NUMBER, + null, 0L))); + Assert.assertTrue(errorDispatcher.offer(new SenderError( + SenderError.Category.UNKNOWN, SenderError.Policy.RETRIABLE, + SenderError.NO_STATUS_BYTE, null, SenderError.NO_MESSAGE_SEQUENCE, + -1L, -1L, null, 0L))); + Assert.assertTrue(progressDispatcher.offer(0L)); + Thread connectionDispatcherThread = connectionDispatcher.getWorkerThreadForTesting(); + Thread errorDispatcherThread = errorDispatcher.getWorkerThreadForTesting(); + Thread progressDispatcherThread = progressDispatcher.getWorkerThreadForTesting(); + Assert.assertNotNull(connectionDispatcherThread); + Assert.assertNotNull(errorDispatcherThread); + Assert.assertNotNull(progressDispatcherThread); + + AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = new Thread(() -> { + Thread.currentThread().interrupt(); + try { + sender.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "full-cleanup-closer"); + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("closer thread did not finish", closer.isAlive()); + Assert.assertNotNull("close() must surface the failed I/O-thread stop", + closeFailure.get()); + + releaseConnect.countDown(); + Thread ioThread = ioThreadRef.get(); + Assert.assertNotNull(ioThread); + ioThread.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("I/O thread did not exit after release", ioThread.isAlive()); + Assert.assertTrue("complete sender cleanup callback did not finish", + sender.isCloseCleanupComplete()); + Assert.assertTrue("loop-owned WebSocket client was not closed", loopClient.isClosed()); + Assert.assertTrue("sender-owned WebSocket client was not closed", senderClient.isClosed()); + Assert.assertFalse("connection dispatcher worker was not reclaimed", + connectionDispatcherThread.isAlive()); + Assert.assertFalse("error dispatcher worker was not reclaimed", + errorDispatcherThread.isAlive()); + Assert.assertFalse("progress dispatcher worker was not reclaimed", + progressDispatcherThread.isAlive()); + Assert.assertTrue("engine cleanup did not complete", engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull(probe); + } + } finally { + releaseConnect.countDown(); + Thread.interrupted(); + if (loop != null) { + try { + loop.close(); + } catch (Throwable ignored) { + } + } + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + // ------------------------------------------------------------------ utils private static void freeFieldQuietly(Object target, String name) { @@ -528,16 +644,27 @@ private static void setField(Object target, String name, Object value) throws Ex * it (close is idempotent via the superclass). */ private static final class StubWebSocketClient extends WebSocketClient { + private final AtomicBoolean isClosed = new AtomicBoolean(); StubWebSocketClient() { super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); } + @Override + public void close() { + isClosed.set(true); + super.close(); + } + @Override protected void ioWait(int timeout, int op) { throw new UnsupportedOperationException("stub: no socket"); } + boolean isClosed() { + return isClosed.get(); + } + @Override protected void setupIoWait() { // no-op diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java new file mode 100644 index 00000000..d6c3f665 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java @@ -0,0 +1,310 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class CursorSendEngineCrashConsistencyTest { + + @Test + public void testCloseDurabilityOrderAndSyncFailurePropagation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (int failAt : new int[]{-1, 0, 1, 2, 4}) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-close-crash-" + failAt + "-" + System.nanoTime()).toString(); + String slot = root + "/slot"; + SegmentManager manager = null; + CursorSendEngine engine = null; + long payload = 0; + Throwable failure = null; + try { + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + CrashImageFilesFacade ff = new CrashImageFilesFacade(slot, failAt); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60), + SegmentManager.UNLIMITED_TOTAL_BYTES, ff); + payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + engine = new CursorSendEngine(slot, segmentSize, manager); + Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7); + Assert.assertEquals(0L, engine.appendBlocking(payload, 32)); + Assert.assertTrue(engine.acknowledge(0L)); + ff.beginClose(); + try { + engine.close(); + if (failAt >= 0 && failAt != 3) { + Assert.fail("sync failure was swallowed at boundary " + failAt); + } + } catch (IllegalStateException expected) { + Assert.assertTrue("unexpected close failure: " + expected, + failAt >= 0 && failAt != 3); + } + engine = null; + + if (failAt >= 0 && failAt <= 2) { + Assert.assertFalse("segment deletion started after watermark barrier failure", + ff.events.contains("segment-remove")); + Assert.assertTrue("watermark was removed after its durability barrier failed", + Files.exists(slot + "/" + AckWatermark.FILE_NAME)); + } else { + Assert.assertFalse("simulated crash replays acknowledged rows at boundary " + failAt, + ff.durableSegments && !ff.durableWatermark); + } + if (failAt == -1) { + Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync", + "dir-fsync", "segment-remove", "dir-fsync", "watermark-remove"), + ff.events); + } + } catch (Throwable t) { + failure = t; + } finally { + failure = closeEngine(failure, engine); + failure = closeManager(failure, manager); + failure = freePayload(failure, payload); + failure = removeRoot(failure, root); + } + rethrow(failure); + } + }); + } + + private static Throwable addCleanupFailure(Throwable failure, Throwable cleanupFailure) { + if (failure == null) { + return cleanupFailure; + } + if (failure != cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + return failure; + } + + private static Throwable closeEngine(Throwable failure, CursorSendEngine engine) { + if (engine != null) { + try { + engine.close(); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static Throwable closeManager(Throwable failure, SegmentManager manager) { + if (manager != null) { + try { + manager.close(); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static Throwable freePayload(Throwable failure, long payload) { + if (payload != 0) { + try { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static Throwable removeRoot(Throwable failure, String root) { + try { + removeRecursive(root); + Assert.assertFalse("test directory was not removed: " + root, Files.exists(root)); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + return failure; + } + + private static void rethrow(Throwable failure) { + if (failure != null) { + CursorSendEngineCrashConsistencyTest.throwUnchecked(failure); + } + } + + @SuppressWarnings("unchecked") + private static void throwUnchecked(Throwable failure) throws T { + throw (T) failure; + } + + private static final class CrashImageFilesFacade implements FilesFacade { + private final List events = new ArrayList<>(); + private final int failAt; + private final String slot; + private boolean active; + private boolean durableSegments = true; + private boolean durableWatermark; + private int eventIndex; + private int watermarkFd = -1; + + private CrashImageFilesFacade(String slot, int failAt) { + this.slot = slot; + this.failAt = failAt; + } + + private void beginClose() { + active = true; + events.clear(); + eventIndex = 0; + } + + private boolean fail(String event) { + events.add(event); + return eventIndex++ == failAt; + } + + @Override + public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override + public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override + public int close(int fd) { return INSTANCE.close(fd); } + @Override + public boolean exists(String path) { return INSTANCE.exists(path); } + @Override + public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override + public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override + public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override + public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override + public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override + public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override + public int fsync(int fd) { + return active && fd == watermarkFd && fail("watermark-fsync") ? -1 : INSTANCE.fsync(fd); + } + @Override + public int fsyncDir(String dir) { + if (active && slot.equals(dir)) { + if (fail("dir-fsync")) return -1; + if (events.contains("segment-remove")) { + durableSegments = false; + } else { + durableWatermark = true; + } + } + return INSTANCE.fsyncDir(dir); + } + @Override + public long length(int fd) { return INSTANCE.length(fd); } + @Override + public long length(String path) { return INSTANCE.length(path); } + @Override + public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override + public int lock(int fd) { return INSTANCE.lock(fd); } + @Override + public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int msync(long addr, long len, boolean async) { + return active && fail("watermark-msync") ? -1 : INSTANCE.msync(addr, len, async); + } + @Override + public int openCleanRW(String path) { + int fd = INSTANCE.openCleanRW(path); + if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override + public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override + public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override + public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + @Override + public boolean remove(String path) { + if (active && path.endsWith(".sfa")) { + if (fail("segment-remove")) return false; + } else if (active && path.equals(slot + "/" + AckWatermark.FILE_NAME)) { + if (fail("watermark-remove")) return false; + } + return INSTANCE.remove(path); + } + @Override + public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override + public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override + public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java index 6c08aeff..2c5f76c3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -85,14 +85,15 @@ public void tearDown() { /** * The structural guarantee: while the manager worker is provably still - * inside a service pass for the engine's ring, {@code close()} must NOT - * hand the slot to anyone else. With the quiescence barrier reverted, - * close() releases the slot lock immediately and the mid-test - * {@code SlotLock.acquire} probe succeeds. Once the pass finishes, its - * deferred cleanup must release the slot without a direct close retry. + * inside a service pass for the engine's ring, repeated direct + * {@code close()} calls must NOT hand the slot to anyone else. With the + * duplicate-cleanup-owner branch reverted, the second close runs cleanup + * inline and the mid-test {@code SlotLock.acquire} probe succeeds. Once + * the pass finishes, its deferred cleanup must run exactly once and + * release the slot without another close retry. */ @Test(timeout = 30_000L) - public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { + public void testRepeatedCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { TestUtils.assertMemoryLeak(() -> { long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); String slot = tmpDir + "/slot"; @@ -103,11 +104,15 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { CountDownLatch workerBlocked = new CountDownLatch(1); CountDownLatch releaseWorker = new CountDownLatch(1); AtomicBoolean fired = new AtomicBoolean(); + AtomicInteger cleanupCount = new AtomicInteger(); AtomicReference hookErr = new AtomicReference<>(); boolean managerClosed = false; CursorSendEngine engine = null; try { - manager.setAfterRingCleanupHook(cleanupFinished::countDown); + manager.setAfterRingCleanupHook(() -> { + cleanupCount.incrementAndGet(); + cleanupFinished.countDown(); + }); manager.setBeforeInstallSyncHook(() -> { if (!fired.compareAndSet(false, true)) return; workerBlocked.countDown(); @@ -136,6 +141,15 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { Assert.assertFalse("incomplete close must remain observable to the owner", engine.isCloseCompleted()); + // Exercise CursorSendEngine.close() directly again while the + // same pass already owns deferredClose. Sender.close() cannot + // reach this branch because its second call is a no-op. + engine.close(); + Assert.assertFalse("repeated close must not steal deferred cleanup ownership", + engine.isCloseCompleted()); + Assert.assertEquals("cleanup ran while the worker pass was blocked", + 0, cleanupCount.get()); + // The slot must still be locked: a replacement engine (or raw // SlotLock) acquiring it now would race the stale worker. try { @@ -154,6 +168,8 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { releaseWorker.countDown(); Assert.assertTrue("ring pass did not finish deferred cleanup", cleanupFinished.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("ring-pass cleanup must run exactly once", + 1, cleanupCount.get()); Assert.assertTrue("ring-pass cleanup must report complete cleanup", engine.isCloseCompleted()); engine = null; @@ -181,7 +197,7 @@ public void testCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { } /** - * Owned-manager twin of {@link #testCloseRetainsSlotWhileWorkerIsMidServicePass}: + * Owned-manager twin of {@link #testRepeatedCloseRetainsSlotWhileWorkerIsMidServicePass}: * the ONLY construction shape production uses (Sender.build, BackgroundDrainer, * QwpWebSocketSender.connect all own their manager). The owned close path does * not run the per-ring barrier at all — it relies on {@code manager.close()}'s diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java new file mode 100644 index 00000000..44a348f5 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java @@ -0,0 +1,529 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + *******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class SegmentManagerCrashConsistencyTest { + + private static void awaitTrimmed(SegmentRing ring) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (ring.firstSealed() != null) { + if (System.nanoTime() > deadline) { + throw new AssertionError("manager did not trim acknowledged segments"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static void awaitValue(AtomicInteger value, int expected, String message) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (value.get() < expected) { + if (System.nanoTime() > deadline) { + throw new AssertionError(message + " [expected=" + expected + ", actual=" + value.get() + ']'); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static SegmentRing createRing(String root, long segmentSize, long payload, int sealedCount) { + SegmentRing ring = null; + boolean success = false; + try { + ring = new SegmentRing(MmapSegment.create(root + "/sf-0.sfa", 0, segmentSize), segmentSize); + Assert.assertEquals(0L, ring.appendOrFsn(payload, 1)); + for (int i = 1; i <= sealedCount; i++) { + ring.installHotSpare(MmapSegment.create(root + "/sf-" + i + ".sfa", i, segmentSize)); + Assert.assertEquals(i, ring.appendOrFsn(payload, 1)); + } + ring.installHotSpare(MmapSegment.create(root + "/sf-" + (sealedCount + 1) + ".sfa", + sealedCount + 1L, segmentSize)); + Assert.assertTrue(ring.acknowledge(sealedCount - 1L)); + success = true; + return ring; + } finally { + if (!success && ring != null) ring.close(); + } + } + + private static AckWatermark openWatermark(FilesFacade ff, String root) throws Exception { + Method method = AckWatermark.class.getDeclaredMethod("open", FilesFacade.class, String.class); + method.setAccessible(true); + return (AckWatermark) method.invoke(null, ff, root); + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(dir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + @Test(timeout = 15_000L) + public void testBackgroundTrimDurabilityOrder() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-order-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync", "dir-fsync", + "segment-remove", "segment-remove", "dir-fsync"), ff.events); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testBarrierFailuresPreserveCrashSafetyAndRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (int failAt : new int[]{0, 1, 2, 5}) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-fault-" + failAt + "-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root, failAt); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + ff.awaitFailure(); + if (failAt == 5) { + Assert.assertNotNull("post-unlink barrier failure committed ring removal", + ring.firstSealed()); + ff.advance(TimeUnit.SECONDS.toNanos(2)); + manager.wakeWorker(); + awaitTrimmed(ring); + } + manager.close(); + } + if (failAt <= 2) { + Assert.assertEquals("unlink started before covering barrier failed", 0, ff.removeCalls.get()); + Assert.assertNotNull("barrier failure removed ring bookkeeping", ring.firstSealed()); + } else { + Assert.assertNull("post-unlink barrier retry did not commit ring removal", ring.firstSealed()); + } + Assert.assertFalse("segment deletion began without a durable covering watermark", + ff.removeCalls.get() > 0 && !ff.durableWatermark); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + } + }); + } + + @Test(timeout = 15_000L) + public void testDiskTrimWithoutWatermarkIsPreserved() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-no-watermark-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 1); + CountDownLatch servicePass = new CountDownLatch(1); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L)) { + manager.setBeforeTrimSyncHook(servicePass::countDown); + manager.register(ring, root, null); + manager.start(); + Assert.assertTrue("manager did not reach the trim service pass", + servicePass.await(5, TimeUnit.SECONDS)); + } + Assert.assertNotNull(ring.firstSealed()); + Assert.assertTrue(Files.exists(root + "/sf-0.sfa")); + } finally { + if (ring != null) ring.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testMoreThanOneQuantumBatchesBarriers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-batch-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 65); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 70L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals(65, ff.removeCalls.get()); + Assert.assertEquals("watermark must sync once per quantum", 2, ff.msyncCalls.get()); + Assert.assertEquals("directory barriers must be twice per quantum", 4, ff.dirSyncCalls.get()); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testPersistentFailureDoesNotStarveSibling() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String base = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-retry-sibling-" + System.nanoTime()).toString(); + String badRoot = base + "/bad"; + String goodRoot = base + "/good"; + Assert.assertEquals(0, Files.mkdir(base, Files.DIR_MODE_DEFAULT)); + Assert.assertEquals(0, Files.mkdir(badRoot, Files.DIR_MODE_DEFAULT)); + Assert.assertEquals(0, Files.mkdir(goodRoot, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(badRoot, "dir-fsync", 0); + AckWatermark badWatermark = null; + AckWatermark goodWatermark = null; + SegmentRing badRing = null; + SegmentRing goodRing = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + badRing = createRing(badRoot, segmentSize, payload, 1); + goodRing = createRing(goodRoot, segmentSize, payload, 1); + badWatermark = openWatermark(ff, badRoot); + goodWatermark = openWatermark(ff, goodRoot); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 16L, ff, ff::ticks)) { + manager.register(badRing, badRoot, badWatermark); + manager.register(goodRing, goodRoot, goodWatermark); + ff.active = true; + manager.start(); + awaitValue(ff.failureCalls, 1, "bad sibling failure was not attempted"); + awaitTrimmed(goodRing); + Assert.assertNotNull("failed sibling unexpectedly trimmed", badRing.firstSealed()); + } + } finally { + if (badRing != null) badRing.close(); + if (goodRing != null) goodRing.close(); + if (badWatermark != null) badWatermark.close(); + if (goodWatermark != null) goodWatermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(badRoot); + removeRecursive(goodRoot); + removeRecursive(base); + } + }); + } + + @Test(timeout = 30_000L) + public void testPersistentFailuresBackOffAndRecover() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String[] failures = {"watermark-msync", "segment-remove", "post-dir-fsync"}; + for (String failure : failures) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-retry-" + failure + '-' + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade( + root, failure, "segment-remove".equals(failure) ? 2 : 0); + if ("watermark-msync".equals(failure)) { + ff.ticks.set(Long.MAX_VALUE - 2_000_000L); + } + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + AtomicInteger logs = new AtomicInteger(); + AtomicInteger passes = new AtomicInteger(); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.setBeforeTrimSyncHook(passes::incrementAndGet); + manager.setRetryLogHook(logs::incrementAndGet); + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitValue(ff.failureCalls, 1, "initial persistent failure was not attempted"); + awaitValue(logs, 1, "initial failure transition was not logged"); + if ("segment-remove".equals(failure)) { + Assert.assertEquals("successful unlink prefix was not committed", 1L, + ring.firstSealed().baseSeq()); + Assert.assertFalse(Files.exists(root + "/sf-0.sfa")); + Assert.assertTrue(Files.exists(root + "/sf-1.sfa")); + } + + int operations = ff.operationCalls(); + for (int i = 0; i < 4; i++) { + int nextPass = passes.get() + 1; + manager.wakeWorker(); + awaitValue(passes, nextPass, "deferred retry pass did not run"); + } + Assert.assertEquals("deferred passes performed filesystem work", + operations, ff.operationCalls()); + Assert.assertEquals("deferred passes emitted logs", 1, logs.get()); + + long delay = 4_000_000L; + for (int attempt = 2; attempt <= 11; attempt++) { + int nextPass = passes.get() + 1; + operations = ff.operationCalls(); + ff.advance(delay - 1); + manager.wakeWorker(); + awaitValue(passes, nextPass, "pre-deadline pass did not run"); + Assert.assertEquals("pre-deadline pass performed filesystem work", + operations, ff.operationCalls()); + nextPass = passes.get() + 1; + ff.advance(1); + manager.wakeWorker(); + awaitValue(ff.failureCalls, attempt, "retry deadline did not enable attempt"); + awaitValue(passes, nextPass, "retry pass did not run"); + Assert.assertEquals("persistent failure log was not throttled", 1, logs.get()); + delay = Math.min(delay * 2, 1_024_000_000L); + } + + ff.failureEnabled = false; + ff.advance(delay); + manager.wakeWorker(); + awaitTrimmed(ring); + awaitValue(logs, 2, "recovery transition was not logged"); + Assert.assertEquals("recovery transition was not logged once", 2, logs.get()); + } + Assert.assertNull(ring.firstSealed()); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + } + }); + } + + private static final class OrderingFilesFacade implements FilesFacade { + private final AtomicInteger dirSyncCalls = new AtomicInteger(); + private final List events = new ArrayList<>(); + private final int failAt; + private final AtomicInteger failureCalls = new AtomicInteger(); + private final AtomicInteger msyncCalls = new AtomicInteger(); + private final AtomicInteger removeCalls = new AtomicInteger(); + private final String persistentEvent; + private final int persistentRemoveOrdinal; + private final String root; + private final AtomicLong ticks = new AtomicLong(); + private boolean active; + private boolean durableSegments = true; + private boolean durableWatermark; + private int eventIndex; + private boolean expectPreDirSync = true; + private volatile boolean failureEnabled; + private volatile boolean failureObserved; + private int watermarkFd = -1; + + private OrderingFilesFacade(String root) { + this(root, -1); + } + + private OrderingFilesFacade(String root, int failAt) { + this.root = root; + this.failAt = failAt; + this.persistentEvent = null; + this.persistentRemoveOrdinal = 0; + } + + private OrderingFilesFacade(String root, String persistentEvent, int persistentRemoveOrdinal) { + this.root = root; + this.failAt = -1; + this.persistentEvent = persistentEvent; + this.persistentRemoveOrdinal = persistentRemoveOrdinal; + this.failureEnabled = true; + } + + private void advance(long nanos) { + ticks.addAndGet(nanos); + } + + private void awaitFailure() { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!failureObserved) { + if (System.nanoTime() > deadline) { + throw new AssertionError("injected barrier failure was not reached"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private boolean fail(String event) { + events.add("post-dir-fsync".equals(event) ? "dir-fsync" : event); + boolean persistentMatch = failureEnabled && event.equals(persistentEvent) + && (!"segment-remove".equals(event) || persistentRemoveOrdinal <= removeCalls.get()); + boolean failed = eventIndex++ == failAt || persistentMatch; + if (failed) { + failureCalls.incrementAndGet(); + failureObserved = true; + } + return failed; + } + + private int operationCalls() { + return msyncCalls.get() + dirSyncCalls.get() + removeCalls.get(); + } + + private long ticks() { + return ticks.get(); + } + + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { return INSTANCE.close(fd); } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { + if (active && fd == watermarkFd && fail("watermark-fsync")) return -1; + return INSTANCE.fsync(fd); + } + @Override public int fsyncDir(String dir) { + if (active && root.equals(dir)) { + dirSyncCalls.incrementAndGet(); + String event = expectPreDirSync ? "dir-fsync" : "post-dir-fsync"; + boolean failed = fail(event); + if (expectPreDirSync) { + if (!failed) { + durableWatermark = true; + expectPreDirSync = false; + } + } else { + expectPreDirSync = true; + if (!failed) durableSegments = false; + } + if (failed) return -1; + } + return INSTANCE.fsyncDir(dir); + } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public int msync(long addr, long len, boolean async) { + if (active) { + msyncCalls.incrementAndGet(); + if (fail("watermark-msync")) return -1; + } + return INSTANCE.msync(addr, len, async); + } + @Override public int openCleanRW(String path) { + int fd = INSTANCE.openCleanRW(path); + if (path.equals(root + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (path.equals(root + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { + if (active && path.endsWith(".sfa")) { + removeCalls.incrementAndGet(); + if (fail("segment-remove")) return false; + } + return INSTANCE.remove(path); + } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java index 0c102fce..a00faea4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; @@ -36,6 +37,10 @@ import org.junit.Test; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -73,6 +78,65 @@ public void tearDown() { Files.remove(tmpDir); } + @Test + public void testAckBatchDoesNotDelaySiblingSpareProvisioning() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int sealedCount = 128; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + CountDownLatch busyDrained = new CountDownLatch(1); + CountDownLatch siblingInstall = new CountDownLatch(1); + AtomicInteger trimPass = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + try (SegmentRing busyRing = new SegmentRing(MmapSegment.createInMemory(0, segSize), segSize); + SegmentRing siblingRing = new SegmentRing(MmapSegment.createInMemory(0, segSize), segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60))) { + for (int i = 0; i < sealedCount; i++) { + assertEquals(i, busyRing.appendOrFsn(buf, 1)); + busyRing.installHotSpare(MmapSegment.createInMemory(busyRing.nextSeqHint(), segSize)); + } + assertEquals(sealedCount, busyRing.appendOrFsn(buf, 1)); + busyRing.installHotSpare(MmapSegment.createInMemory(busyRing.nextSeqHint(), segSize)); + busyRing.acknowledge(sealedCount - 1L); + + manager.register(busyRing, null); + manager.register(siblingRing, null); + manager.setBeforeInstallSyncHook(() -> { + try { + assertEquals("the busy ring must stop at the 64-segment quantum before sibling service", + 64, busyRing.getSealedSegments().size()); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } finally { + siblingInstall.countDown(); + } + }); + manager.setBeforeTrimSyncHook(() -> { + if (trimPass.incrementAndGet() == 4) { + try { + assertEquals("the next fair pass must fully drain the busy ring", + 0, busyRing.getSealedSegments().size()); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } finally { + busyDrained.countDown(); + } + } + }); + manager.start(); + assertTrue("manager did not reach sibling spare provisioning", + siblingInstall.await(10, TimeUnit.SECONDS)); + assertTrue("manager parked instead of immediately rescheduling the remaining ACK batch", + busyDrained.await(10, TimeUnit.SECONDS)); + if (failure.get() != null) { + throw new AssertionError("ACK batch fairness witness failed", failure.get()); + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testManagerProvisionsSpareWithinPollingTick() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -136,10 +200,12 @@ public void testManagerTrimsAckedSegmentFiles() throws Exception { String seg0Path = tmpDir + "/0000000000000000.sfa"; MmapSegment seg0 = MmapSegment.create(seg0Path, 0, segSize); long buf = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); - try (SegmentRing ring = new SegmentRing(seg0, segSize); + try (AckWatermark watermark = AckWatermark.open(tmpDir); + SegmentRing ring = new SegmentRing(seg0, segSize); SegmentManager mgr = new SegmentManager(segSize, 200_000L)) { + assertTrue("watermark must open", watermark != null); mgr.start(); - mgr.register(ring, tmpDir); + mgr.register(ring, tmpDir, watermark); // Fill seg0 (2 frames) and force rotation by appending a third. for (int i = 0; i < 2; i++) ring.appendOrFsn(buf, 32); @@ -170,11 +236,13 @@ public void testMaxTotalBytesCapBlocksProvisioningUntilTrimFrees() throws Except long cap = 3 * segSize; MmapSegment seg0 = MmapSegment.create(tmpDir + "/0000000000000000.sfa", 0, segSize); long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); - try (SegmentRing ring = new SegmentRing(seg0, segSize); + try (AckWatermark watermark = AckWatermark.open(tmpDir); + SegmentRing ring = new SegmentRing(seg0, segSize); SegmentManager mgr = new SegmentManager(segSize, 200_000L, cap)) { + assertTrue("watermark must open", watermark != null); mgr.start(); // register seeds totalBytes = 1*segSize (initial active). - mgr.register(ring, tmpDir); + mgr.register(ring, tmpDir, watermark); // Manager provisions spare 1 → totalBytes = 2*segSize. assertTrue(waitFor(() -> !ring.needsHotSpare(), 2000)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java index 8cc05407..e622ac1b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java @@ -43,7 +43,6 @@ import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -186,11 +185,8 @@ public void testTrimPathDoesNotDoubleSubtractAfterDeregister() throws Exception + "`totalBytes -= sz` on a stillRegistered re-check " + "under the same lock that covers deregister.", 0L, observed); - assertFalse("stale SegmentManager snapshot skipped drainTrimmable() " - + "after deregister and left a fully-acked sealed " - + "segment on disk. The registration guard should " - + "protect watermark/accounting only; trim ownership " - + "transfer must still close and unlink " + activePath, + assertTrue("deregister before the durable barrier must preserve the segment " + + "for owner-side quiescent cleanup " + activePath, Files.exists(activePath)); } finally { mgr.setBeforeTrimSyncHook(null); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java index 0feffe8a..67e0be65 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java @@ -138,8 +138,8 @@ public void testFailedUnlinkRetainsBookkeepingAndUsesSuccessorPath() throws Exce Assert.assertNotNull("failed unlink removed the segment from ring bookkeeping", ring.firstSealed()); Assert.assertEquals(failedPath, ring.firstSealed().path()); - Assert.assertEquals("watermark advanced although unlink did not commit", - -1L, watermark.read()); + Assert.assertEquals("failed unlink must remain covered by the durable cumulative watermark", + 0L, watermark.read()); fill(payload, 32, (byte) 0x5A); Assert.assertEquals("non-DEDUP successor must continue at the next FSN", diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index f1a0fcde..0d31320b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -38,6 +38,8 @@ import org.junit.Test; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -428,9 +430,10 @@ public void testAcknowledgeIsMonotonic() throws Exception { } @Test - public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws Exception { + public void testNextSealedAfterWalksThousandsOfSegmentsInLinearOperations() throws Exception { TestUtils.assertMemoryLeak(() -> { - // Regression for "sealed snapshot grew unexpectedly large". + // Regression for both "sealed snapshot grew unexpectedly large" + // and a later quadratic list traversal at every segment boundary. // The cursor I/O loop used to copy the entire sealed list into a // fixed-size array (initial 16, grown once to 32) on every advance. // Under load — producer outpacing the WS sender, no maxTotalBytes @@ -462,8 +465,18 @@ public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws // After the loop we have `sealedCount` sealed segments and one // active (containing nothing yet — its base = sealedCount). // Now walk: oldest sealed, then nextSealedAfter() repeatedly. + SegmentRing.resetNextSealedComparisons(); MmapSegment cursor = ring.firstSealed(); assertNotNull(cursor); + for (int i = 1; i < sealedCount / 2; i++) { + cursor = ring.nextSealedAfter(cursor); + assertNotNull(cursor); + } + long halfWalkOperations = SegmentRing.getNextSealedComparisons(); + + SegmentRing.resetNextSealedComparisons(); + cursor = ring.firstSealed(); + assertNotNull(cursor); assertEquals(0, cursor.baseSeq()); int visited = 1; long prevBase = cursor.baseSeq(); @@ -478,8 +491,14 @@ public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws visited++; } assertEquals("must visit every sealed segment", sealedCount, visited); - // Walking past the last sealed → null (caller falls through to active). - assertNull(ring.nextSealedAfter(cursor)); + // The loop terminated when walking past the last sealed returned null. + long comparisons = SegmentRing.getNextSealedComparisons(); + assertTrue("walk inspected " + comparisons + " entries for " + sealedCount + + " segments; successor traversal must remain O(N)", + comparisons <= 2L * sealedCount); + assertTrue("doubling the walk grew operations from " + halfWalkOperations + + " to " + comparisons + "; expected linear scaling", + comparisons <= 2L * halfWalkOperations + 2L); } } finally { Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); @@ -528,6 +547,73 @@ public void testNextSealedAfterStillReturnsCorrectlyWhenCursorWasTrimmed() throw }); } + @Test(timeout = 30_000L) + public void testNextSealedAfterSurvivesConcurrentRotationAndHeadTrim() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 16); + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment seg0 = MmapSegment.create(tmpDir + "/race-0.sfa", 0, segSize); + try (SegmentRing ring = new SegmentRing(seg0, segSize)) { + fillPattern(buf, 16, 0); + for (int i = 0; i < 3; i++) { + assertEquals(i, ring.appendOrFsn(buf, 16)); + ring.installHotSpare(MmapSegment.create( + tmpDir + "/race-" + (i + 1) + ".sfa", + ring.nextSeqHint(), segSize)); + } + MmapSegment cursor = ring.firstSealed(); + assertNotNull(cursor); + ring.acknowledge(1); + + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + Thread rotate = new Thread(() -> { + ready.countDown(); + try { + start.await(); + assertEquals(3L, ring.appendOrFsn(buf, 16)); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }, "segment-rotate"); + Thread trim = new Thread(() -> { + ready.countDown(); + try { + start.await(); + ObjList drained = ring.drainTrimmable(); + assertNotNull(drained); + assertEquals(2, drained.size()); + for (int i = 0; i < drained.size(); i++) { + drained.get(i).close(); + } + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }, "segment-trim"); + + synchronized (ring) { + rotate.start(); + trim.start(); + ready.await(); + start.countDown(); + } + rotate.join(); + trim.join(); + assertNull("concurrent mutation failed: " + failure.get(), failure.get()); + + MmapSegment next = ring.nextSealedAfter(cursor); + assertNotNull(next); + assertEquals("trim fallback must skip both removed successors", 2L, next.baseSeq()); + assertEquals("rotation must leave the former active sealed", 1, ring.getSealedSegments().size()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + /** * Open-time sort regression: at the documented {@code sf_max_total_bytes * / sf_max_bytes} ceiling (~16K segments) an O(N²) sort over the @@ -600,6 +686,52 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { }); } + @Test + public void testRemovingAcknowledgedPrefixMovesLinearReferences() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + long previousMoves = -1; + for (int sealedCount = 64; sealedCount <= 256; sealedCount *= 2) { + MmapSegment initial = MmapSegment.createInMemory(0, segSize); + try (SegmentRing ring = new SegmentRing(initial, segSize)) { + for (int i = 0; i < 2 * sealedCount; i++) { + assertEquals(i, ring.appendOrFsn(buf, 1)); + ring.installHotSpare(MmapSegment.createInMemory(ring.nextSeqHint(), segSize)); + } + assertEquals(2L * sealedCount, ring.appendOrFsn(buf, 1)); + ring.acknowledge(sealedCount - 1L); + SegmentRing.resetTrimMovedReferences(); + MmapSegment segment; + int removed = 0; + while ((segment = ring.firstTrimmable()) != null) { + assertTrue(ring.removeTrimmable(segment)); + segment.close(); + removed++; + } + assertEquals(sealedCount, removed); + assertEquals("unacknowledged suffix must remain live", sealedCount, + ring.getSealedSegments().size()); + assertEquals(sealedCount, ring.firstSealed().baseSeq()); + long moves = SegmentRing.getTrimMovedReferences(); + assertTrue("removing " + sealedCount + " entries moved " + moves + + " references; expected amortized linear work", + moves <= 2L * sealedCount); + if (previousMoves >= 0) { + assertTrue("doubling the acknowledged prefix grew moved references from " + + previousMoves + " to " + moves, + moves <= 2L * previousMoves + sealedCount); + } + previousMoves = moves; + } + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testMaxBytesPerSegmentReturnsConfiguredValue() throws Exception { // Direct constructor path: the value passed in must round-trip through diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java index 81055bb6..38561c46 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java @@ -32,6 +32,8 @@ import org.junit.Test; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.nio.file.Paths; import java.util.concurrent.atomic.AtomicBoolean; @@ -83,6 +85,75 @@ public void preWarmClosesBuiltDelegatesWhenBuildThrowsError() throws Exception { }); } + @Test(timeout = 30_000) + public void elasticBorrowClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try (SenderPool pool = new SenderPool( + CFG, 0, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, null), false, + () -> { + throw failure; + })) { + try { + pool.borrow(); + Assert.fail("borrow must propagate the post-factory failure"); + } catch (AssertionError actual) { + Assert.assertSame("borrow must preserve throwable identity", failure, actual); + } + Assert.assertEquals("failed elastic creation must close its delegate", 1, closeCalls.get()); + } + }); + } + + @Test(timeout = 30_000) + public void preWarmClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try { + new SenderPool( + CFG, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, null), false, + () -> { + throw failure; + }); + Assert.fail("prewarm must propagate the post-factory failure"); + } catch (AssertionError actual) { + Assert.assertSame("prewarm must preserve throwable identity", failure, actual); + } + Assert.assertEquals("failed prewarm creation must close its delegate", 1, closeCalls.get()); + }); + } + + @Test(timeout = 30_000) + public void recoveryClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AssertionError closeFailure = new AssertionError("injected close failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try (SenderPool pool = new SenderPool( + CFG, 0, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, closeFailure), false, + () -> { + throw failure; + })) { + Method createRecoverer = SenderPool.class.getDeclaredMethod("createRecoverer", int.class); + createRecoverer.setAccessible(true); + try { + createRecoverer.invoke(pool, -1); + Assert.fail("recovery creation must propagate the post-factory failure"); + } catch (InvocationTargetException e) { + Assert.assertSame("recovery must preserve throwable identity", failure, e.getCause()); + Assert.assertArrayEquals("close failure must remain secondary", + new Throwable[]{closeFailure}, failure.getSuppressed()); + } + Assert.assertEquals("failed recovery creation must close its delegate", 1, closeCalls.get()); + } + }); + } + // Companion to the catch (RuntimeException) -> track-normal-completion fix in // PooledSender.close(). flush() can exit with an Error (AssertionError under // -ea, OutOfMemoryError, ...) as well as a RuntimeException; the wrapper is @@ -232,6 +303,42 @@ public void borrowReleasesSfSlotIndexWhenCreationFails() throws Exception { }); } + private static Sender closeCountingSender(AtomicInteger closeCalls, Throwable closeFailure) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + if ("close".equals(method.getName())) { + closeCalls.incrementAndGet(); + if (closeFailure != null) { + throw closeFailure; + } + return null; + } + if ("toString".equals(method.getName())) { + return "CloseCountingSender"; + } + if ("hashCode".equals(method.getName())) { + return System.identityHashCode(proxy); + } + if ("equals".equals(method.getName())) { + return proxy == args[0]; + } + Class rt = method.getReturnType(); + if (rt == boolean.class) return false; + if (rt == byte.class) return (byte) 0; + if (rt == short.class) return (short) 0; + if (rt == int.class) return 0; + if (rt == long.class) return 0L; + if (rt == float.class) return 0f; + if (rt == double.class) return 0d; + if (rt == char.class) return (char) 0; + if (rt == void.class) return null; + if (rt.isInstance(proxy)) return proxy; + return null; + }); + } + private static Sender fakeSender(AtomicBoolean closedFlag) { return (Sender) Proxy.newProxyInstance( Sender.class.getClassLoader(), diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 896dc69a..ea7e901a 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Paths; @@ -871,6 +872,105 @@ public void testDeferredFlockReleaseWakesParkedLongTimeoutBorrower() throws Exce }); } + @Test + public void testDirectRetiredSlotCallbacksHaveLinearProbeCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + final int slotCount = 32; + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, slotCount, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[slotCount]; + Sender[] delegates = new Sender[slotCount]; + Runnable[] callbacks = new Runnable[slotCount]; + for (int i = 0; i < slotCount; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + callbacks[i] = (Runnable) getField(delegates[i], "slotLockReleaseListener"); + Assert.assertNotNull(callbacks[i]); + } + for (int i = 0; i < slotCount; i++) { + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals(slotCount, pool.leakedSlotCount()); + setLongField(pool, "retiredSlotProbeCount", 0); + + int[] geometricCheckpoints = {4, 8, 16, 32}; + int checkpoint = 0; + for (int i = 0; i < slotCount; i++) { + setBooleanField(delegates[i], "slotLockReleased", true); + callbacks[i].run(); + if (i + 1 == geometricCheckpoints[checkpoint]) { + Assert.assertEquals("direct release probes must grow linearly", + i + 1, getLongField(pool, "retiredSlotProbeCount")); + checkpoint++; + } + } + Assert.assertEquals(0, pool.leakedSlotCount()); + Assert.assertTrue(((List) getField(pool, "retiredSlots")).isEmpty()); + } + } + }); + } + + @Test + public void testDirectRetiredSlotCallbackFallbackAndIdempotence() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 0, 3, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[3]; + Sender[] delegates = new Sender[3]; + Runnable[] callbacks = new Runnable[3]; + for (int i = 0; i < 3; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + callbacks[i] = (Runnable) getField(delegates[i], "slotLockReleaseListener"); + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals(3, pool.leakedSlotCount()); + setLongField(pool, "retiredSlotProbeCount", 0); + + // Simulate callback registration becoming unavailable. The + // periodic housekeeper scan must remain a complete fallback. + ((QwpWebSocketSender) delegates[0]).setSlotLockReleaseListener(null); + setBooleanField(delegates[0], "slotLockReleased", true); + pool.reapIdle(); + Assert.assertEquals(2, pool.leakedSlotCount()); + + // A premature callback must not remove an unreleased slot. + callbacks[1].run(); + Assert.assertEquals(2, pool.leakedSlotCount()); + setBooleanField(delegates[1], "slotLockReleased", true); + callbacks[1].run(); + Assert.assertEquals(1, pool.leakedSlotCount()); + + // Duplicate and stale callbacks are idempotent and do not + // probe or mutate the slot after its direct removal. + setBooleanField(delegates[2], "slotLockReleased", true); + callbacks[2].run(); + long probesAfterRecovery = getLongField(pool, "retiredSlotProbeCount"); + callbacks[2].run(); + callbacks[1].run(); + Assert.assertEquals(probesAfterRecovery, + getLongField(pool, "retiredSlotProbeCount")); + Assert.assertEquals(0, pool.leakedSlotCount()); + Assert.assertTrue(((List) getField(pool, "retiredSlots")).isEmpty()); + } + } + }); + } + @Test public void testMixedRetiredSlotsRecoverWithoutLosingAccounting() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -2373,6 +2473,85 @@ public void testFailedOutOfRangeRecoveryRetriesAfterPrimaryReturns() throws Exce }); } + @Test + public void testDrainFailureRetriesInRangeAndOutOfRangeCandidates() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + for (int i = 0; i < 2; i++) { + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-" + i + ";close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", i).atNow(); + seed.flush(); + } + } + } + Assert.assertTrue("in-range fixture must contain unacked data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + IntFunction factory = idx -> { + if (attempts[idx].incrementAndGet() == 1) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + if ("drain".equals(method.getName())) { + throw new LineSenderException("transient drain failure for slot " + idx); + } + if ("close".equals(method.getName())) { + return null; + } + throw new AssertionError("unexpected recovery sender call: " + method.getName()); + }); + } + return Sender.builder(config).senderId("default-" + idx).build(); + }; + + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { + Assert.assertFalse("failed in-range drain must defer the same candidate", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals(1, attempts[0].get()); + Assert.assertEquals(0, attempts[1].get()); + Assert.assertTrue("failed in-range drain must preserve durable data", + hasSegmentFile(slot("default-0"))); + + Assert.assertTrue("successful in-range retry may continue scanning", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals("same live pool must retry the in-range candidate", + 2, attempts[0].get()); + Assert.assertFalse("in-range retry must drain the preserved data", + hasSegmentFile(slot("default-0"))); + + Assert.assertFalse("failed out-of-range drain must defer the same candidate", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals(1, attempts[1].get()); + Assert.assertTrue("failed out-of-range drain must preserve durable data", + hasSegmentFile(slot("default-1"))); + + Assert.assertTrue("successful out-of-range retry may finish the candidate", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals("same live pool must retry the out-of-range candidate", + 2, attempts[1].get()); + Assert.assertFalse("out-of-range retry must drain the preserved data", + hasSegmentFile(slot("default-1"))); + Assert.assertFalse("final scan step must mark recovery complete", + invokeRunStartupRecoveryStep(pool)); + Assert.assertTrue("both recovered frames must be delivered", handler.frames.get() >= 2); + } + } + }); + } + @Test public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws Exception { // The drain exclusion is bounded to [0, maxSize) so a sibling's drainer @@ -3135,12 +3314,24 @@ private static void setBooleanField(Object target, String name, boolean value) t f.setBoolean(target, value); } + private static void setLongField(Object target, String name, long value) throws Exception { + Field f = target.getClass().getDeclaredField(name); + f.setAccessible(true); + f.setLong(target, value); + } + private static int getIntField(Object target, String name) throws Exception { Field f = target.getClass().getDeclaredField(name); f.setAccessible(true); return f.getInt(target); } + private static long getLongField(Object target, String name) throws Exception { + Field f = target.getClass().getDeclaredField(name); + f.setAccessible(true); + return f.getLong(target); + } + private static Object getField(Object target, String name) throws Exception { Field f = target.getClass().getDeclaredField(name); f.setAccessible(true); diff --git a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java index 47d168f9..6ae57621 100644 --- a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java +++ b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java @@ -34,7 +34,9 @@ import io.questdb.client.network.Socket; import io.questdb.client.network.SocketReadinessWaiter; import io.questdb.client.network.TlsSessionInitFailedException; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Os; +import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Assume; import org.junit.Test; @@ -262,6 +264,90 @@ public void testPlainSocketShutdownWakesMacOsKqueueAndRetainsFd() throws Excepti assertShutdownWakesMacOsKqueue(new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class))); } + @Test(timeout = 30_000L) + public void testPlainSocketShutdownWakesWindowsRecvAndRetainsFd() throws Exception { + Assume.assumeTrue("real Winsock cancellation coverage runs on Windows", Os.type == Os.WINDOWS); + + Socket socket = new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + AtomicBoolean recvDone = new AtomicBoolean(); + AtomicInteger recvResult = new AtomicInteger(Integer.MIN_VALUE); + AtomicReference recvFailure = new AtomicReference<>(); + CountDownLatch recvStarted = new CountDownLatch(1); + + long buffer = 0; + int fd = -1; + Thread waiter = null; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfo(fd, addrInfo)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept()) { + socket.of(fd); + int retainedFd = fd; + fd = -1; + buffer = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + long recvBuffer = buffer; + + waiter = new Thread(() -> { + recvStarted.countDown(); + try { + recvResult.set(socket.recv(recvBuffer, 1)); + } catch (Throwable t) { + recvFailure.set(t); + } finally { + recvDone.set(true); + } + }, "socket-traffic-windows-recv-waiter"); + waiter.setDaemon(true); + waiter.start(); + + Assert.assertTrue("waiter did not reach the receive call", + recvStarted.await(5, TimeUnit.SECONDS)); + Assert.assertFalse("peer unexpectedly made the receive complete", recvDone.get()); + + socket.closeTraffic(); + + waiter.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("shutdown did not wake the native receive", waiter.isAlive()); + Assert.assertNull("native receive failed", recvFailure.get()); + Assert.assertTrue("shutdown must disconnect the native receive", recvResult.get() < 0); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + Assert.assertTrue("shutdown must leave the Winsock descriptor allocated", NF.getSndBuf(retainedFd) > 0); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + Assert.assertEquals("released Winsock descriptor must reject socket operations", -1, NF.getSndBuf(retainedFd)); + } + } finally { + if (waiter != null && waiter.isAlive()) { + try { + socket.closeTraffic(); + } catch (Throwable ignored) { + // Full close below is the final wake-up fallback. + } + } + socket.close(); + if (waiter != null) { + waiter.join(TimeUnit.SECONDS.toMillis(5)); + } + if (buffer != 0 && (waiter == null || !waiter.isAlive())) { + Unsafe.free(buffer, 1, MemoryTag.NATIVE_DEFAULT); + } + if (fd != -1) { + NF.close(fd); + } + } + } + @Test public void testTlsSocketTrafficGatePreservesTlsStateUntilFullClose() throws Exception { AtomicInteger closeCount = new AtomicInteger(); diff --git a/core/src/test/java/io/questdb/client/test/std/ObjListTest.java b/core/src/test/java/io/questdb/client/test/std/ObjListTest.java index b2efff7b..cc8c6e8b 100644 --- a/core/src/test/java/io/questdb/client/test/std/ObjListTest.java +++ b/core/src/test/java/io/questdb/client/test/std/ObjListTest.java @@ -28,6 +28,8 @@ import org.junit.Assert; import org.junit.Test; +import java.lang.reflect.Field; + /** * */ @@ -58,6 +60,21 @@ public void testRemoveFromTo() { Assert.assertEquals(list(), remove(list("a", "b", "c"), 4, 10)); } + @Test + public void testRemoveFromToClearsFinalBackingSlot() throws Exception { + ObjList values = new ObjList<>(16); + for (int i = 0; i < 16; i++) { + values.add(new Object()); + } + + values.remove(0, 0); + + Field bufferField = ObjList.class.getDeclaredField("buffer"); + bufferField.setAccessible(true); + Object[] buffer = (Object[]) bufferField.get(values); + Assert.assertNull(buffer[buffer.length - 1]); + } + private static ObjList remove(ObjList o, int from, int to) { o.remove(from, to); return o; From b8d965d53d8e6490cf1a772201b254fd17b3e049 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 12 Jul 2026 22:05:06 +0100 Subject: [PATCH 18/26] Make socket traffic shutdown idempotent Treat ENOTCONN and WSAENOTCONN as successful traffic shutdown because the peer may finish a graceful close before the owner cancels the I/O thread. Preserve all other failures and descriptor ownership. Add real loopback coverage for peer-first close and a synthetic failure case that verifies non-benign shutdown errors remain visible. --- core/src/main/c/share/net.c | 3 +- core/src/main/c/windows/net.c | 5 ++ .../network/SocketTrafficShutdownTest.java | 78 +++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/core/src/main/c/share/net.c b/core/src/main/c/share/net.c index 9b58fd65..f3bc1ddd 100644 --- a/core/src/main/c/share/net.c +++ b/core/src/main/c/share/net.c @@ -130,7 +130,8 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_send JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown (JNIEnv *e, jclass cl, jint fd) { - return shutdown((int) fd, SHUT_RDWR); + const int result = shutdown((int) fd, SHUT_RDWR); + return result == -1 && errno == ENOTCONN ? 0 : result; } JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv diff --git a/core/src/main/c/windows/net.c b/core/src/main/c/windows/net.c index 9cc98000..d105b504 100644 --- a/core/src/main/c/windows/net.c +++ b/core/src/main/c/windows/net.c @@ -234,6 +234,11 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown (JNIEnv *e, jclass cl, jint fd) { const int result = shutdown((SOCKET) fd, SD_BOTH); if (result == SOCKET_ERROR) { + const int error = WSAGetLastError(); + if (error == WSAENOTCONN) { + return 0; + } + WSASetLastError(error); SaveLastError(); } return result; diff --git a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java index 6ae57621..3a737f50 100644 --- a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java +++ b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java @@ -259,6 +259,84 @@ public void testCompatibilityDefaultsDoNotBypassCustomTransportOwnership() { 0, socketCloseCount.get()); } + @Test(timeout = 30_000L) + public void testPlainSocketShutdownAfterPeerDisconnectRetainsFd() throws Exception { + Socket socket = new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + + long buffer = 0; + int fd = -1; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfo(fd, addrInfo)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept()) { + socket.of(fd); + int retainedFd = fd; + fd = -1; + buffer = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + + peer.close(); + Assert.assertTrue("client must observe the peer disconnect", socket.recv(buffer, 1) < 0); + + socket.closeTraffic(); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + Assert.assertTrue("shutdown must leave the descriptor allocated", NF.getSndBuf(retainedFd) > 0); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + Assert.assertEquals("released descriptor must reject socket operations", -1, NF.getSndBuf(retainedFd)); + } + } finally { + socket.close(); + if (buffer != 0) { + Unsafe.free(buffer, 1, MemoryTag.NATIVE_DEFAULT); + } + if (fd != -1) { + NF.close(fd); + } + } + } + + @Test + public void testPlainSocketShutdownFailureStillThrows() { + AtomicInteger closeCount = new AtomicInteger(); + NetworkFacade failingFacade = new CompatibilityNetworkFacade(closeCount) { + @Override + public int errno() { + return 1234; + } + + @Override + public int shutdown(int fd) { + Assert.assertEquals(42, fd); + return -1; + } + }; + PlainSocket socket = new PlainSocket(failingFacade, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + socket.of(42); + + try { + socket.closeTraffic(); + Assert.fail("expected genuine traffic shutdown failure"); + } catch (IllegalStateException expected) { + Assert.assertEquals("could not shut down socket traffic [fd=42, errno=1234]", expected.getMessage()); + } finally { + socket.close(); + } + + Assert.assertTrue(socket.isClosed()); + Assert.assertEquals("full close must release facade ownership exactly once", 1, closeCount.get()); + } + @Test(timeout = 30_000L) public void testPlainSocketShutdownWakesMacOsKqueueAndRetainsFd() throws Exception { assertShutdownWakesMacOsKqueue(new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class))); From 79eb13c4b67397b6f968704372bd2bc304a3316a Mon Sep 17 00:00:00 2001 From: bluestreak Date: Mon, 13 Jul 2026 18:05:15 +0100 Subject: [PATCH 19/26] fix(client): serialize QuestDB.close() through shutdown completion QuestDBImpl.close() set the volatile `closed` flag before running the pool teardown chain, so a second concurrent close() caller could observe closed==true and return while the first caller was still draining and releasing pool resources -- a premature return that breaks the AutoCloseable expectation that shutdown has completed once close() returns. Make close() synchronized so the losing caller blocks on the monitor until the winner finishes teardown, then enters, sees `closed`, and no-ops. A bare CAS is insufficient: the losing caller would still return early. Adds a latch-controlled two-closer regression test (QuestDBImplCloseTest) that is red without the fix. --- .../io/questdb/client/impl/QuestDBImpl.java | 14 +- .../test/impl/QuestDBImplCloseTest.java | 191 ++++++++++++++++++ 2 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java index e3da539b..3545d3fa 100644 --- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java @@ -181,8 +181,20 @@ public Sender borrowSender() { return senderPool.borrow(); } + // synchronized so concurrent close() callers serialize THROUGH shutdown + // completion, not merely through the `closed` flip. `closed` is set before + // the teardown chain runs, so a plain volatile guard (or a bare CAS) would + // let a second caller observe closed==true and return while the first is + // still inside closeQuietly(senderPool) releasing the flock/mmap/I/O-thread + // resources -- a premature return that breaks the AutoCloseable contract + // that shutdown has completed once close() returns. The monitor makes the + // losing caller block until the winner finishes, then it enters, sees + // `closed` and returns a no-op. No deadlock: the teardown steps + // (markClosing/housekeeper.stop()/queryPool.close()/senderPool.close()) + // never call back into QuestDBImpl.close() on another thread, so nothing + // contends for this monitor from within the critical section. @Override - public void close() { + public synchronized void close() { if (closed) { return; } diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java new file mode 100644 index 00000000..c4a33e9c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java @@ -0,0 +1,191 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.impl; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.impl.QuestDBImpl; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.IntFunction; + +/** + * P2 regression: {@link QuestDBImpl#close()} must not return before shutdown + * has completed, even when two threads call it concurrently. {@code closed} is + * volatile and set BEFORE the pool teardown chain runs, so a naive guard lets a + * second concurrent caller observe {@code closed == true} and RETURN while the + * first caller is still inside {@code closeQuietly(senderPool)} tearing down the + * flock/mmap/I/O-thread resources. After ANY {@code close()} returns -- the + * losing concurrent caller included -- callers must be able to assume shutdown + * has completed. + *

    + * The window is opened deterministically by injecting (via the {@code @TestOnly} + * senderFactory seam) a fake delegate whose {@code close()} parks on a latch. + * The last teardown step, {@code senderPool.close()}, closes the prewarmed idle + * delegate on the closing thread OUTSIDE the pool lock, so thread A parks there + * with {@code closed} already raised. Thread B then calls {@code close()}: + *

      + *
    • pre-fix -- B reads {@code closed == true} and returns immediately while A + * is still tearing down (premature return);
      • + *
    • fixed -- B blocks until A finishes the teardown, then returns.
      • + *
    + * Latch-coordinated (no {@code Thread.sleep} for correctness) with a JUnit + * timeout on the two-thread interleaving. + */ +public class QuestDBImplCloseTest { + + // Non-SF http config: the injected senderFactory replaces the native build, + // but the constructor's eager config probe must still parse it. + private static final String QUERY_CFG = "ws::addr=127.0.0.1:9000;"; + private static final String SENDER_CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + + // RED (closed set before teardown, no serialization): while thread A is + // parked inside the delegate teardown of the final senderPool.close() step, + // thread B's close() sees closed==true and RETURNS -- closeReturnedEarly is + // true and the first assertion fails. GREEN (close() serialized through + // completion): B blocks on A until the teardown finishes, so its close() + // does not return until delegateCloses == 1. + @Test(timeout = 30_000) + public void concurrentCloseSecondCallerBlocksUntilShutdownCompletes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger delegateCloses = new AtomicInteger(); + CountDownLatch inDelegateClose = new CountDownLatch(1); + CountDownLatch releaseDelegateClose = new CountDownLatch(1); + IntFunction senderFactory = slotIndex -> + parkingCloseSender(delegateCloses, inDelegateClose, releaseDelegateClose); + // queryMin = 0 -> QueryClientPool prewarms nothing, so the connect + // hook is never reached and its teardown is a no-op; the parking + // delegate is the only blocking teardown step. + Consumer connectHook = client -> { + }; + + QuestDBImpl questDB = newQuestDB(senderFactory, connectHook); + + // Thread A: enter close() and park inside the final teardown step + // (senderPool.close() -> idle delegate close()), with closed + // already raised. + Thread closerA = new Thread(questDB::close, "questdb-closer-A"); + closerA.start(); + Assert.assertTrue("closer A never reached the delegate teardown", + inDelegateClose.await(10, TimeUnit.SECONDS)); + + // Thread B: a concurrent close(). It must NOT return while A is + // still tearing down. + Thread closerB = new Thread(questDB::close, "questdb-closer-B"); + closerB.start(); + closerB.join(300); + boolean closeReturnedEarly = !closerB.isAlive(); + int closesWhenBReturned = delegateCloses.get(); + + // Always unpark the teardown so the test fails on the assertion, not + // its own timeout. + releaseDelegateClose.countDown(); + + Assert.assertFalse( + "concurrent close() returned while the first caller was still tearing down " + + "(delegateCloses=" + closesWhenBReturned + " when B returned): " + + "close() must serialize through shutdown completion", + closeReturnedEarly); + + // Once the teardown completes, B must return promptly and the + // delegate must have been torn down exactly once. + closerB.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("concurrent close() did not return after the teardown completed", + closerB.isAlive()); + closerA.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("first close() did not return after the teardown completed", + closerA.isAlive()); + Assert.assertEquals("the prewarmed delegate must be torn down exactly once", + 1, delegateCloses.get()); + }); + } + + private static QuestDBImpl newQuestDB( + IntFunction senderFactory, Consumer connectHook + ) { + return new QuestDBImpl( + SENDER_CFG, QUERY_CFG, + /*senderMin*/ 1, /*senderMax*/ 1, + /*queryMin*/ 0, /*queryMax*/ 1, + /*acquireTimeoutMillis*/ 250L, + /*idleTimeoutMillis*/ Long.MAX_VALUE, + /*maxLifetimeMillis*/ Long.MAX_VALUE, + /*housekeeperIntervalMillis*/ Long.MAX_VALUE, + senderFactory, connectHook); + } + + /** + * Proxy-backed fake Sender whose {@code close()} signals {@code inClose}, + * parks on {@code releaseClose}, then bumps {@code closes} -- a delegate + * teardown frozen mid-close so the test can probe what a concurrent + * close() does while it runs. + */ + private static Sender parkingCloseSender( + AtomicInteger closes, + CountDownLatch inClose, + CountDownLatch releaseClose + ) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + inClose.countDown(); + if (!releaseClose.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("test never released the parked close"); + } + closes.incrementAndGet(); + return null; + case "toString": + return "ParkingCloseFakeSender"; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + Class rt = method.getReturnType(); + if (rt == boolean.class) return false; + if (rt == byte.class) return (byte) 0; + if (rt == short.class) return (short) 0; + if (rt == int.class) return 0; + if (rt == long.class) return 0L; + if (rt == float.class) return 0f; + if (rt == double.class) return 0d; + if (rt == char.class) return (char) 0; + if (rt == void.class) return null; + if (rt.isInstance(proxy)) return proxy; + return null; + } + }); + } +} From e8c2dcd8ef36a660175580787eb9e3f252e0e8f1 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Mon, 13 Jul 2026 21:32:35 +0100 Subject: [PATCH 20/26] fix(qwp): cancel in-flight connect during close, with a bounded backstop CursorWebSocketSendLoop.close() did an untimed shutdownLatch.await() while the I/O thread could be blocked in an in-flight foreground connect (connect_timeout=0 => OS SYN-retry ~60-130s). The connecting WebSocketClient is a walk-local in QwpWebSocketSender.connectWalk, invisible to close() (the `client` field is null on async-initial connect / stale on reconnect), so closeTraffic() could not reach it and close() hung -- risking Sender.close() exceeding the sidecar's 120s deadline. Publish a race-safe per-loop cancellation handle (ConnectCancellation) through the transport seam: connectWalk publishes the client it is about to block on before connect(); close() calls closeTraffic() on it to unwind a black-holed connect. The ReconnectFactory seam gains a Java-8 `default reconnect(ConnectCancellation)`, so every existing implementor stays source/binary compatible. Add a bounded backstop: close() awaits the shutdown latch for at most DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS (30s, ~4x under the sidecar deadline) and, on timeout, runs the same loud failed-stop protocol as the interrupt branch (delegates final teardown to the I/O thread's exit path, frees nothing under the live worker) -- so close() returns bounded even in the rare TOCTOU window where cancellation is a no-op. Also clear the in-flight handle on connect-failure paths so it never dangles at a disposed client. Adds CursorWebSocketSendLoopConnectPhaseCloseTest (async-initial + mid-flight cancellation, plus bounded-backstop-without-cancellation). --- .../qwp/client/QwpWebSocketSender.java | 58 ++- .../sf/cursor/CursorWebSocketSendLoop.java | 195 +++++++- ...ebSocketSendLoopConnectPhaseCloseTest.java | 469 ++++++++++++++++++ 3 files changed, 715 insertions(+), 7 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8300cedc..3f272eb9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -2768,22 +2768,37 @@ private WebSocketClient newWebSocketClient() { * reconnect and {@code close()} paths are therefore never queued * behind a drainer's endpoint walk. */ - private WebSocketClient buildAndConnect(ReconnectSupplier ctx) { + private WebSocketClient buildAndConnect(ReconnectSupplier ctx, CursorWebSocketSendLoop.ConnectCancellation cancellation) { if (ctx.isBackground()) { // Lock-free: the walk below touches only internally-synchronized // hostTracker health state and walk-local/cursor-local state on // the background path. - return connectWalk(ctx); + return connectWalk(ctx, cancellation); } connectWalkLock.lock(); try { - return connectWalk(ctx); + return connectWalk(ctx, cancellation); } finally { connectWalkLock.unlock(); } } - private WebSocketClient connectWalk(ReconnectSupplier ctx) { + // Drop the in-flight connect handle once the walk has disposed a client on + // a connect/upgrade FAILURE, so inFlight never dangles at a disposed client + // across the inter-attempt backoff. Without this a concurrent + // close()->cancel() could closeTraffic() a client the walk no longer owns; + // proven a no-op on both production transports today (closed-fd closeTraffic + // no-ops), but a future custom transport that threw on a closed socket would + // spuriously loud-fail close(). Null-guarded: the no-arg reconnect() path + // and the Unsafe.allocateInstance bare-loop tests pass a null handle. Pairs + // with the success-path clear() after upgrade(). + private static void clearInFlight(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + if (cancellation != null) { + cancellation.clear(); + } + } + + private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLoop.ConnectCancellation cancellation) { // Background (drainer) factories share this connect walk -- endpoint // list and hostTracker HEALTH state (never the shared round: a // background sweep walks its own RoundCursor and records with @@ -2871,9 +2886,34 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpRequestDurableAck(requestDurableAck); newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs)); + if (cancellation != null) { + // Publish the client we are about to block on so a + // concurrent CursorWebSocketSendLoop.close() can break its + // traffic and unwind a black-holed native connect + // (connect_timeout=0 => OS SYN-retry) instead of hanging on + // the untimed shutdown-latch await. The publish-then-check + // handshake pairs with ConnectCancellation.cancel(): if we + // observe cancellation here we skip the blocking connect + // entirely; otherwise cancel() observed this client and + // breaks it. The walk's per-attempt catch disposes the + // client and, since running has flipped false, the + // top-of-loop ctx.isAborted() gate ends the walk. + cancellation.publish(newClient); + if (cancellation.isCancelled()) { + throw new LineSenderException(ctx.abortMessage()); + } + } newClient.connect(ep.host, ep.port); int upgradeTimeoutMs = (int) Math.min(authTimeoutMs, Integer.MAX_VALUE); newClient.upgrade(WRITE_PATH, upgradeTimeoutMs, authorizationHeader); + if (cancellation != null) { + // connect()+upgrade() completed: this client is no longer + // blocking, so drop it from the in-flight handle before it + // becomes the loop's `client` field. close() must then break + // its traffic via the field path exactly once -- leaving it + // in the handle too would double-shut-down the socket. + cancellation.clear(); + } } catch (HttpClientException e) { // Close BEFORE classify: the sibling catch (Error) below does not // guard catch-arm bodies, so an Error thrown inside classify() @@ -2883,6 +2923,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { // upgradeStatusCode) that are set during upgrade() and survive // close(). newClient.close(); + clearInFlight(cancellation); HttpClientException classified = QwpUpgradeFailures.classify(newClient, ep.host, ep.port, e); if (classified instanceof QwpIngressRoleRejectedException) { QwpIngressRoleRejectedException re = (QwpIngressRoleRejectedException) classified; @@ -2929,6 +2970,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { continue; } catch (Exception e) { newClient.close(); + clearInFlight(cancellation); hostTracker.recordTransportError(idx, !background); lastError = e; if (!background) { @@ -2951,6 +2993,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { // the cursor reconnect loop, BackgroundDrainer) rethrows Error // rather than retrying, so this stays a loud one-shot failure. closeQuietlyOnError(newClient); + clearInFlight(cancellation); throw e; } // Guard the post-upgrade tail: from here until newClient is @@ -4008,7 +4051,12 @@ boolean isAborted() { @Override public WebSocketClient reconnect() { - return buildAndConnect(this); + return buildAndConnect(this, null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + return buildAndConnect(this, cancellation); } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index b8ae47d9..97b2d7d0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; @@ -88,6 +89,29 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * sub-second confirmation latency once the upload completes * server-side. {@code 0} or negative disables the keepalive entirely. */ + /** + * Bounded-await backstop for {@link #close()}: the maximum time close() + * waits for the I/O thread to stop (count down {@code shutdownLatch}) + * before it loud-fails and delegates final teardown to the I/O thread's + * exit path. In the common case the round-2 in-flight connect cancellation + * (see {@link ConnectCancellation}) makes the I/O thread unwind and count + * the latch down within milliseconds, so this budget is never reached. It + * only fires in the pathological, astronomically-rare TOCTOU window where + * {@code cancel()}'s {@code closeTraffic()} lands between the pre-connect + * guard and native fd creation, so it is a no-op and the connect blocks the + * full OS SYN-retry (~60-130s per endpoint, possibly across several). + *

    + * {@code 30_000} ms is comfortably UNDER the sidecar's 120 s shutdown + * deadline (~4x headroom) yet ~1000x a healthy close's millisecond latch + * countdown, so it never prematurely abandons a legitimately draining + * close (the I/O thread only has to finish its current — now traffic-broken + * — native send/receive and run {@code ioLoop}'s finally). This bounds the + * WAIT inside close(), NOT the connect itself (rejected Option A): a + * legitimately slow SUCCESSFUL connect is unaffected — it is not concurrent + * with a close, and when close() does fire it cancels the in-flight connect + * rather than waiting it out. + */ + public static final long DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS = 30_000L; public static final long DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS = 200L; public static final long DEFAULT_PARK_NANOS = 50_000L; // 50us idle backoff /** @@ -163,6 +187,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // the durable watermark can lag behind the OK watermark. private final ArrayDeque pendingDurable = new ArrayDeque<>(); private final ArrayDeque pendingDurablePool = new ArrayDeque<>(); + // Race-safe cancellation handle for an in-flight connect attempt. Passed + // into reconnectFactory.reconnect(...) on the I/O thread so close() can + // break a connect blocked mid-attempt (a black-holed native connect that + // neither unpark nor interrupt cancels). See ConnectCancellation. + private final ConnectCancellation connectCancellation = new ConnectCancellation(); // Optional reconnect plumbing. When non-null, a wire failure triggers a // reconnect attempt instead of a terminal fail(). The factory produces a // fresh, connected+upgraded WebSocketClient. @@ -225,6 +254,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // it is engine.ackedFsn() + 1, so the first replayed frame on the new // connection is wireSeq=0 and server-side cumulative ACKs still line up. private long fsnAtZero; + // Bounded-await backstop budget for close() (see + // DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS). Overridable via + // setShutdownAwaitTimeoutMillis so tests can exercise the timeout branch + // deterministically without a multi-second real wait; production always + // uses the default. Read only on the owner thread inside close(). + private long shutdownAwaitTimeoutMillis = DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS; // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a // successful connect attempt in any mode). Once true, stays true. Used to @@ -783,8 +818,55 @@ public synchronized void close() { ); } } + // Cancel an in-flight connect attempt not yet installed as the + // `client` field: async initial connect leaves `client` null + // and a mid-flight reconnect leaves it pointing at the stale + // pre-drop client, so neither is reachable by the closeTraffic() + // above. The reconnect factory publishes the connecting client + // here before it blocks, so breaking its traffic unwinds a + // black-holed native connect (connect_timeout=0 => OS SYN-retry, + // ~60-130s) that would otherwise pin the untimed await below. + // Same loud-fail contract as the field client: a transport that + // cannot safely shut down traffic is not destructively closed; + // the worker's exit path retains ownership of final cleanup. try { - shutdownLatch.await(); + connectCancellation.cancel(); + } catch (Throwable e) { + throw new LineSenderException( + "cursor I/O thread did not stop: active transport does not support safe traffic shutdown; " + + "client/engine teardown is delegated to the I/O thread's exit path", + e + ); + } + try { + // Bounded backstop: never wait forever. The round-2 in-flight + // connect cancellation makes the I/O thread unwind and count + // the latch down in milliseconds, so this await returns true + // almost always. The timeout branch fires only in the + // pathological uninterruptible-connect TOCTOU window where + // cancel()'s closeTraffic() was a no-op and the connect blocks + // the full OS SYN-retry -- guaranteeing close() still returns + // (bounded, comfortably under the sidecar's 120s deadline) + // rather than hanging on an untimed await. + if (!shutdownLatch.await(shutdownAwaitTimeoutMillis, TimeUnit.MILLISECONDS)) { + // Latch still up after the budget: the I/O thread did not + // stop. Same failed-stop protocol as the interrupt branch + // below -- loud-fail without touching the client/engine + // (freeing native buffers under a still-live I/O thread + // risks a C5 SEGV; a quiet return would let the owner + // unmap the engine under it). The I/O thread's own exit + // path (ioLoop's finally) retains ownership of final + // cleanup; QwpWebSocketSender.close() keys its + // ioThreadStopped guard on this throw and BackgroundDrainer + // switches to delegateEngineClose(). ioThread stays set + // (not nulled below, since we throw) so a duplicate close() + // re-signals rather than silently succeeding. + throw new LineSenderException( + "cursor I/O thread did not stop: close() timed out after " + + shutdownAwaitTimeoutMillis + "ms awaiting shutdown; " + + "client/engine teardown is delegated to the I/O " + + "thread's exit path"); + } } catch (InterruptedException e) { // Re-assert the flag for the caller's stack, then decide. // If the I/O thread has genuinely not exited (latch still @@ -996,6 +1078,17 @@ public void setProgressDispatcher(SenderProgressDispatcher dispatcher) { this.progressDispatcher = dispatcher; } + /** + * Test seam: shrink the {@link #close()} bounded-await backstop + * (default {@link #DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS}) so a test can + * exercise the timeout branch deterministically without a multi-second + * real wait. Production never calls this. Set before {@link #close()}. + */ + @TestOnly + public void setShutdownAwaitTimeoutMillis(long millis) { + this.shutdownAwaitTimeoutMillis = millis; + } + public synchronized void start() { if (ioThread != null) { throw new IllegalStateException("already started"); @@ -1172,7 +1265,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM attempts++; totalReconnectAttempts.incrementAndGet(); try { - WebSocketClient newClient = reconnectFactory.reconnect(); + WebSocketClient newClient = reconnectFactory.reconnect(connectCancellation); if (newClient != null) { if (!running) { // close() ran while this connect attempt was in @@ -2049,6 +2142,104 @@ private boolean tryRetireOrphanTail() { @FunctionalInterface public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; + + /** + * Cancellable variant of {@link #reconnect()}. The loop passes a + * per-attempt {@link ConnectCancellation} so a transport that blocks + * inside a native connect can publish the in-flight client to the + * handle BEFORE it blocks; {@link #close()} then breaks that client's + * traffic to unwind a black-holed connect promptly, rather than falling + * back to the bounded {@code shutdownLatch.await(...)} backstop (which + * only bounds the WAIT, not the connect). + *

    + * Default: ignore the handle and delegate to {@link #reconnect()} -- + * a transport that cannot publish its in-flight client is simply not + * cancellable mid-connect, and {@code close()} falls back to its + * existing field-client {@code closeTraffic()} / loud-fail path. The + * default keeps this a {@link FunctionalInterface} and preserves + * source compatibility for existing lambda / method-reference + * implementors (e.g. {@link #connectWithRetry}). + */ + default WebSocketClient reconnect(ConnectCancellation cancellation) throws Exception { + return reconnect(); + } + } + + /** + * Race-safe cancellation handle for a single in-flight connect attempt. + * Owned per-loop and passed into {@link ReconnectFactory#reconnect(ConnectCancellation)} + * on the I/O thread. The transport's connect walk {@link #publish}es the + * {@link WebSocketClient} it is about to block on before the blocking + * {@code connect()}, and {@link #clear}s it once the attempt is installed + * or disposed. {@link #close()} (owner thread) calls {@link #cancel()} to + * break the in-flight client's traffic path so a black-holed native + * connect unwinds and the I/O thread counts down the shutdown latch. + *

    + * Java 8: two volatiles, no locks. Visibility/ordering: + *

      + *
    • {@code inFlight} is written by the I/O thread (publish/clear) and + * read by the owner thread (cancel).
      • + *
    • {@code cancelled} is written by the owner thread (cancel) and read + * by the I/O thread (the pre-connect guard).
      • + *
    + * The publish (write inFlight, then read cancelled) / cancel (write + * cancelled, then read inFlight) handshake covers the close-vs-connect + * race in both directions: if the guard observes {@code cancelled==false} + * then, by the volatile total order, {@code cancel()} observed the + * published client and broke its traffic; if the guard observes + * {@code cancelled==true} it skips the blocking connect entirely. + */ + public static final class ConnectCancellation { + // The client the connect walk is currently about to block / blocked + // on. Written by the I/O thread only (publish/clear); read by the + // owner thread (cancel). volatile for cross-thread visibility. + private volatile WebSocketClient inFlight; + // Latched once close() requested cancellation. Written by the owner + // thread (cancel); read by the I/O thread's pre-connect guard. + private volatile boolean cancelled; + + public boolean isCancelled() { + return cancelled; + } + + /** + * I/O-thread hook: record the client the walk is about to block on, + * BEFORE the blocking {@code connect()}. Pairs with {@link #cancel()} + * so an attempt that publishes after cancellation is caught by the + * caller's {@link #isCancelled()} guard, and an attempt already + * blocked in {@code connect()} is broken by {@code cancel()}. + */ + public void publish(WebSocketClient client) { + inFlight = client; + } + + /** + * I/O-thread hook: the walk is done with the in-flight attempt + * (installed on success, disposed on failure). Drop the reference so a + * later {@link #cancel()} cannot touch a client the walk no longer + * owns. Single writer (I/O thread) for publish/clear, so no CAS is + * needed. + */ + public void clear() { + inFlight = null; + } + + /** + * Owner-thread hook from {@link #close()}: request cancellation, then + * break the traffic path of any in-flight connect so the I/O thread's + * connect walk unwinds. {@code cancelled} is written before reading + * {@code inFlight} to pair with {@link #publish(WebSocketClient)}. May + * throw when the transport cannot safely shut down traffic -- + * {@code close()} maps that to its existing loud-fail, exactly as it + * does for the field client's {@code closeTraffic()}. + */ + void cancel() { + cancelled = true; + WebSocketClient c = inFlight; + if (c != null) { + c.closeTraffic(); + } + } } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java new file mode 100644 index 00000000..0225d071 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java @@ -0,0 +1,469 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * TRUE-CANCELLATION proof (P1): {@code close()} must cancel a connect attempt + * that is blocked inside {@code reconnectFactory.reconnect(...)} (a black-holed + * native connect that neither unpark nor interrupt can cancel). The in-flight + * {@link WebSocketClient} is walk-local -- it is NOT the loop's {@code client} + * field (that field is {@code null} on the async-initial connect and points at + * the stale pre-drop client on a mid-flight reconnect) -- so before the fix + * {@code close()}'s field-client {@code closeTraffic()} could not reach it and + * {@code close()} blocked on the untimed {@code shutdownLatch.await()} for the + * whole OS SYN-retry window (~60-130s per endpoint). + *

    + * The fix publishes the in-flight client to a race-safe + * {@link CursorWebSocketSendLoop.ConnectCancellation} handle before the + * blocking connect, and {@code close()} breaks that client's traffic. These + * tests assert TRUE cancellation, not just return: the fake in-flight client's + * {@code closeTraffic()} is the ONLY thing that unblocks the parked + * {@code reconnect()}, and the tests witness that {@code closeTraffic()} was + * invoked (exactly once, by the closer thread) AND that {@code close()} + * returned. Deterministic latches only; {@code @Test(timeout=...)} fails a + * regression fast. + */ +public class CursorWebSocketSendLoopConnectPhaseCloseTest { + + /** + * Generous budget: with the fix, close() breaks the in-flight connect and + * returns well within this. Without the fix, close() blocks on the untimed + * shutdown latch for the entire (simulated) connect and this budget lapses. + */ + private static final long CLOSE_BUDGET_MILLIS = 5_000L; + + /** + * Small, injectable bounded-await backstop for the TOCTOU test. Shrunk from + * the production {@code DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS} (30 s) via + * {@link CursorWebSocketSendLoop#setShutdownAwaitTimeoutMillis(long)} so the + * timeout branch fires fast and deterministically -- no multi-second real + * wait -- while still leaving CLOSE_BUDGET_MILLIS of slack for the closer + * thread to return. + */ + private static final long BACKSTOP_MILLIS = 500L; + + /** + * Async-initial-connect path: the loop is built with a {@code null} client + * and a reconnect factory, so the I/O thread drives the very first connect + * through connectLoop while the {@code client} field stays {@code null}. + */ + @Test(timeout = 30_000L) + public void testCloseCancelsAsyncInitialConnectViaInFlightClient() throws Exception { + runConnectCancelledByCloseTraffic(false); + } + + /** + * Mid-flight-reconnect path: an initial client is installed, its first + * receive fails, so connectLoop reconnects and blocks. The {@code client} + * field then points at the stale pre-drop client, never the in-flight one. + */ + @Test(timeout = 30_000L) + public void testCloseCancelsMidFlightReconnectViaInFlightClient() throws Exception { + runConnectCancelledByCloseTraffic(true); + } + + /** + * BACKSTOP (bounded-await) proof: models the pathological, uninterruptible + * TOCTOU case where {@code cancel()}'s {@code closeTraffic()} CANNOT break + * the in-flight connect (as if cancellation landed after the pre-connect + * guard but before native fd creation, making closeTraffic() a no-op). The + * connect stays parked, so round-2 cancellation does NOT release the latch. + * Asserts {@code close()} STILL returns within the bounded backstop (does + * not hang) and surfaces the failed-stop contract: a loud + * {@link LineSenderException} whose message names the stalled I/O thread and + * the timeout, with client/engine teardown delegated to the I/O thread's + * own exit path (no destructive close under the still-live worker). + */ + @Test(timeout = 30_000L) + public void testCloseReturnsBoundedWhenCancellationCannotBreakConnect() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final AtomicReference published = new AtomicReference<>(); + final CountDownLatch connectEntered = new CountDownLatch(1); + // The ONLY thing that releases the parked connect. closeTraffic() + // deliberately does NOT touch it, modelling a cancel that cannot + // break the connect; the test itself counts it down, AFTER the + // backstop assertions, so the worker can unwind for the leak check. + final CountDownLatch release = new CountDownLatch(1); + + final CursorWebSocketSendLoop.ReconnectFactory factory = new CursorWebSocketSendLoop.ReconnectFactory() { + @Override + public WebSocketClient reconnect() throws Exception { + return reconnect(null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + UninterruptibleInFlightClient c = new UninterruptibleInFlightClient(release); + published.set(c); + if (cancellation != null) { + cancellation.publish(c); + } + connectEntered.countDown(); + // Uninterruptible AND uncancellable: closeTraffic() does not + // release this park; only the test's release latch does. + c.awaitRelease(); + c.close(); + throw new LineSenderException("connect ended after release"); + } + }; + + final CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + final CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + /* reconnectMaxDurationMillis */ 60_000L, + /* reconnectInitialBackoffMillis */ 1_000L, + /* reconnectMaxBackoffMillis */ 5_000L, + false + ); + // Shrink the bounded-await backstop so the timeout branch fires fast + // (no multi-second real wait); production uses the 30s default. + loop.setShutdownAwaitTimeoutMillis(BACKSTOP_MILLIS); + + final AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = null; + try { + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking connect", + connectEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "backstop-closer"); + closer.start(); + + closer.join(CLOSE_BUDGET_MILLIS); + final boolean closedInTime = !closer.isAlive(); + + final UninterruptibleInFlightClient inFlight = published.get(); + Assert.assertNotNull("no in-flight client was ever published", inFlight); + // cancel() DID attempt to break the connect -- but closeTraffic() + // is a no-op here, so the connect is still parked. + Assert.assertTrue("close() must have attempted to cancel the in-flight connect", + inFlight.trafficCloseCount.get() >= 1); + Assert.assertEquals("the parked connect must NOT have been released by closeTraffic()", + 1L, release.getCount()); + + // Decisive: close() returns within the bounded backstop even + // though cancellation could not break the connect. + Assert.assertTrue( + "close() must return within the bounded backstop (" + CLOSE_BUDGET_MILLIS + + "ms) even when cancellation cannot break the connect; instead it hung", + closedInTime); + + // Failed-stop contract: loud LineSenderException naming the + // stalled thread and the timeout. + final Throwable failure = closeFailure.get(); + Assert.assertNotNull("close() must loud-fail when the backstop times out", failure); + Assert.assertTrue("failed stop must be a LineSenderException, was " + failure, + failure instanceof LineSenderException); + Assert.assertTrue("message must name the stalled I/O thread: " + failure.getMessage(), + failure.getMessage().contains("cursor I/O thread did not stop")); + Assert.assertTrue("message must attribute the failed stop to the backstop timeout: " + + failure.getMessage(), + failure.getMessage().contains("timed out")); + // Teardown is delegated to the I/O thread's exit path, not done + // destructively under the live worker: no terminal manufactured. + Assert.assertNull("backstop timeout must not manufacture a terminal error", + loop.getTerminalError()); + } finally { + // Now let the parked connect unwind so the worker exits and the + // leak check sees a clean teardown. Restore a generous backstop + // so the reconciling close() below awaits the worker's exit. + loop.setShutdownAwaitTimeoutMillis(CLOSE_BUDGET_MILLIS); + release.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + } + }); + } + + private void runConnectCancelledByCloseTraffic(boolean withInitialClient) throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Allocate the initial client INSIDE the leak-checked block so its + // native buffers are inside the baseline, not counted as over-free. + final WebSocketClient initialClient = withInitialClient ? new FailingInitialClient() : null; + final AtomicReference published = new AtomicReference<>(); + final CountDownLatch connectEntered = new CountDownLatch(1); + + // A reconnect factory that models the real connect walk: it creates + // the client it is about to block on, PUBLISHES it to the loop's + // cancellation handle before the blocking connect, then parks in a + // way that ONLY closeTraffic() (i.e. cancellation) can release -- + // exactly a black-holed native connect that unpark/interrupt cannot + // cancel. + final CursorWebSocketSendLoop.ReconnectFactory factory = new CursorWebSocketSendLoop.ReconnectFactory() { + @Override + public WebSocketClient reconnect() throws Exception { + return reconnect(null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + InFlightConnectClient c = new InFlightConnectClient(); + published.set(c); + if (cancellation != null) { + cancellation.publish(c); + } + connectEntered.countDown(); + // Black-holed connect: returns only once closeTraffic() breaks it. + c.awaitTrafficBreak(); + // Cancelled: dispose the half-built client (frees native + // buffers) and surface a transport error, exactly as the + // real connect walk's catch does on a broken connect. + c.close(); + throw new LineSenderException("connect cancelled by closeTraffic()"); + } + }; + + final CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + final CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + initialClient, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + /* reconnectMaxDurationMillis */ 60_000L, + /* reconnectInitialBackoffMillis */ 1_000L, + /* reconnectMaxBackoffMillis */ 5_000L, + false + ); + + final AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = null; + try { + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking connect", + connectEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "connect-phase-closer"); + closer.start(); + + closer.join(CLOSE_BUDGET_MILLIS); + final boolean closedInTime = !closer.isAlive(); + + final InFlightConnectClient inFlight = published.get(); + Assert.assertNotNull("no in-flight client was ever published to the cancellation handle", + inFlight); + + // Let the worker finish unwinding regardless of outcome so the + // memory-leak check sees a clean teardown. + inFlight.trafficBroken.countDown(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + + Assert.assertNull("close() must not fail", closeFailure.get()); + Assert.assertTrue( + "close() must cancel the connect blocked inside reconnect() and return " + + "within " + CLOSE_BUDGET_MILLIS + "ms; instead it blocked on the " + + "untimed shutdown latch for the whole connect", + closedInTime); + // Decisive: closeTraffic() on the IN-FLIGHT client is what + // unblocked the parked connect -- and it was the closer thread, + // not the I/O worker, that broke it. + Assert.assertEquals( + "close() must break the in-flight connect's traffic exactly once", + 1, inFlight.trafficCloseCount.get()); + Assert.assertEquals( + "the closer thread (not the I/O worker) must cancel the in-flight connect", + closer, inFlight.trafficCloseThread.get()); + Assert.assertNull("ordinary connect-phase close must not manufacture a terminal error", + loop.getTerminalError()); + } finally { + InFlightConnectClient inFlight = published.get(); + if (inFlight != null) { + inFlight.trafficBroken.countDown(); + } + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + if (initialClient != null) { + initialClient.close(); + } + } + }); + } + + /** + * The fake in-flight client. It never really connects: the factory parks in + * {@link #awaitTrafficBreak()} until {@link #closeTraffic()} releases it, + * which is the ONLY path out -- neither unpark nor interrupt can cancel it. + */ + private static final class InFlightConnectClient extends WebSocketClient { + final CountDownLatch trafficBroken = new CountDownLatch(1); + final AtomicInteger trafficCloseCount = new AtomicInteger(); + final AtomicReference trafficCloseThread = new AtomicReference<>(); + + private InFlightConnectClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void closeTraffic() { + trafficCloseThread.compareAndSet(null, Thread.currentThread()); + trafficCloseCount.incrementAndGet(); + trafficBroken.countDown(); + } + + void awaitTrafficBreak() { + // Uninterruptible: only closeTraffic()'s countDown may release us, + // faithfully modelling a native connect that unpark/interrupt cannot + // cancel. + boolean interrupted = false; + while (trafficBroken.getCount() != 0L) { + try { + trafficBroken.await(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * The fake in-flight client for the BACKSTOP test. Its {@link #closeTraffic()} + * is a no-op w.r.t. releasing the park -- it only records that cancellation + * was attempted -- so a {@code close()}->{@code cancel()} CANNOT unblock the + * parked connect. This models the pathological uninterruptible/TOCTOU case; + * the ONLY release is the test-owned {@code release} latch, counted down + * AFTER the backstop assertions. + */ + private static final class UninterruptibleInFlightClient extends WebSocketClient { + final AtomicInteger trafficCloseCount = new AtomicInteger(); + private final CountDownLatch release; + + private UninterruptibleInFlightClient(CountDownLatch release) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.release = release; + } + + @Override + public void closeTraffic() { + // Records the cancel attempt but does NOT release the parked + // connect: exactly a closeTraffic() that lands before the native fd + // exists (a no-op), so the connect blocks on regardless. + trafficCloseCount.incrementAndGet(); + } + + void awaitRelease() { + // Uninterruptible: neither unpark, interrupt, nor closeTraffic() + // releases us -- only the test's release latch. + boolean interrupted = false; + while (release.getCount() != 0L) { + try { + release.await(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * A pre-installed client whose first receive fails, driving the I/O loop + * into a reconnect so the in-flight client is published while the + * {@code client} field still points here (the mid-flight-reconnect path). + */ + private static final class FailingInitialClient extends WebSocketClient { + + private FailingInitialClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + throw new LineSenderException("initial wire dropped"); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} From d130f2c9002d0dcbde2d303cc86342b19e402f45 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 14 Jul 2026 00:14:19 +0100 Subject: [PATCH 21/26] fix(client): fail closed in SF recovery and add crash-safe boundary manifest SF crash recovery previously failed OPEN on operational errors: - findFirst() < 0 (permission/transient enumeration error) was treated as an empty directory, so the engine started fresh and openCleanRW(O_TRUNC) truncated the existing durable log (sf-initial.sfa) and deleted the ack watermark; - findNext() < 0 silently accepted an incomplete directory listing; - per-file openRW/mmap failures (EMFILE, ENOMEM, transient I/O) on valid segments were swallowed by the same catch as data corruption and skipped; - the FSN contiguity check only compares adjacent pairs, so a skipped LEADING segment silently dropped unacked rows (ackedFsn seeded past them) and a skipped TRAILING segment silently re-issued its FSNs to new payloads (overlapping FSNs on disk). Recovery now fails closed and proves slot integrity before mutating: - Enumeration errors (findFirst/findNext < 0) abort startup. - Operational open/mmap failures on recognized segments abort startup; only positively-identified corruption (new MmapSegmentCorruptionException: bad magic, sub-header size, negative baseSeq, unreadable header page) is skippable, and quarantine renames (.corrupt) are deferred until the surviving chain validates, so a failed recovery mutates nothing. Unsupported-version segments stay fatal (they belong to another client build; renaming them would strand that build's frames). - sf-manifest.bin: dual-slot, CRC-protected, generation-versioned boundary record (headBase/activeBase) written on rotation and ack-driven trim. Recovery validates the chain against the committed boundaries, which catches missing leading/trailing segments that contiguity alone cannot. Segments carry a manifest-required header flag so a deleted manifest next to migrated segments fails closed. Boundary updates are monotonic (clamped) and serialized with trim under the ring monitor; the trim path recomputes the successor under that monitor so a concurrent rotation can never make the head leapfrog a still-unacked sealed segment. - Segment files are created with O_EXCL (new Files.openRWExclusive natives) instead of O_TRUNC, so no code path can truncate an existing log. - Crash-window protocol, verified state by state: fresh start orders initial-segment durability -> manifest create -> flag stamp; rotation syncs the promoted spare's rebased header before the manifest references it and commits the manifest before any ring mutation; clean-drain close durably collapses boundaries to head==active before unlinking and removes the manifest last. Every window either recovers (equivalent empties from a fresh-start kill, an empty active at the chain end after a rotation crash, record-less manifest creation debris, drain-window survivors, stale below-head files) or fails closed with intact evidence. The one deliberate mutation on a failing path is quarantining a provably record-less manifest (it contains no boundaries by construction; flagged segments still fail closed afterwards). Regression tests (SegmentRecoveryIntegrityTest, 24 cases) cover the full required matrix: findFirst=-1, partial findNext=-1, openRW=-1, mmap failure -- each asserting byte-for-byte slot preservation; the engine-level O_TRUNC reproduction; leading/trailing/interior boundary loss; corrupt stray quarantine with sibling recovery; deferred-quarantine no-mutation on failure; flagged-segments-without-manifest; manifest creation-crash debris self-heal (zero-byte and record-less); drain-crash spare survivor; corrupt-active-with-empty-stand-in fail-closed; dual-slot generation selection and torn-record fallback. Independently reviewed for crash-consistency holes, lock ordering (manager lock -> ring monitor -> manifest monitor, acyclic), unacked-frame loss, and fd/mmap leaks; both review blockers (creation-debris brick, drain-window spare brick) are fixed and regression-tested. Windows native openRWExclusive0 uses CREATE_NEW; CI rebuilds platform binaries from the .c change. --- core/src/main/c/share/files.c | 7 + core/src/main/c/windows/files.c | 9 + .../client/sf/cursor/CursorSendEngine.java | 123 ++- .../qwp/client/sf/cursor/MmapSegment.java | 79 +- .../MmapSegmentCorruptionException.java | 50 ++ .../sf/cursor/MmapSegmentException.java | 8 +- .../qwp/client/sf/cursor/SegmentManager.java | 33 +- .../qwp/client/sf/cursor/SegmentRing.java | 665 ++++++++++---- .../qwp/client/sf/cursor/SfManifest.java | 282 ++++++ .../client/std/DefaultFilesFacade.java | 20 + .../java/io/questdb/client/std/Files.java | 20 + .../io/questdb/client/std/FilesFacade.java | 16 + .../qwp/client/sf/cursor/MmapSegmentTest.java | 43 +- .../cursor/SegmentRecoveryIntegrityTest.java | 827 ++++++++++++++++++ .../cursor/SegmentRingRecoveryUnlinkTest.java | 23 +- 15 files changed, 1975 insertions(+), 230 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java diff --git a/core/src/main/c/share/files.c b/core/src/main/c/share/files.c index fefd24fb..01e18a6b 100644 --- a/core/src/main/c/share/files.c +++ b/core/src/main/c/share/files.c @@ -65,6 +65,13 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRW0 return (jint) fd; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRWExclusive0 + (JNIEnv *e, jclass cl, jlong lpszName) { + int fd; + RESTARTABLE(open((const char *) (uintptr_t) lpszName, O_CREAT | O_EXCL | O_RDWR, 0644), fd); + return (jint) fd; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openAppend0 (JNIEnv *e, jclass cl, jlong lpszName) { int fd; diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index 620bd81e..7e57de94 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -103,6 +103,15 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRW0 FILE_ATTRIBUTE_NORMAL); } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRWExclusive0 + (JNIEnv *e, jclass cl, jlong lpszName) { + return open_file((const char *) (uintptr_t) lpszName, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL); +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openAppend0 (JNIEnv *e, jclass cl, jlong lpszName) { jint fd = open_file((const char *) (uintptr_t) lpszName, diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 1d46c489..9e606b94 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -286,9 +286,11 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // session before deciding to start fresh. Without this the engine // would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs // already on disk and corrupting ACK translation, trim, and replay. - SegmentRing recovered = memoryMode ? null - : SegmentRing.openExisting(sfDir, segmentSizeBytes); - this.wasRecoveredFromDisk = recovered != null; + SegmentRing.Recovery recovery = memoryMode ? null + : SegmentRing.recover(filesFacade, sfDir, segmentSizeBytes); + SegmentRing recovered = recovery == null ? null : recovery.ring(); + this.wasRecoveredFromDisk = recovery != null + && recovery.status() == SegmentRing.RecoveryStatus.RECOVERED; if (recovered != null) { ringInProgress = recovered; // Seed ackedFsn to one below the lowest segment's baseSeq. @@ -394,18 +396,49 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } MmapSegment initial; String initialPath = null; + SfManifest initialManifest = null; if (memoryMode) { initial = MmapSegment.createInMemory(0L, segmentSizeBytes); } else { + // Created WITHOUT the manifest-required flag: the flag is + // a durable promise that sf-manifest.bin exists, so it + // must only be stamped after the manifest itself is on + // disk. Stamping it at create time would leave a crash + // window (initial durable, manifest not yet created) + // whose recovery hard-fails with "segment requires + // missing manifest" even though nothing was lost. initialPath = sfDir + "/sf-initial.sfa"; - initial = MmapSegment.create(initialPath, 0L, segmentSizeBytes); + initial = MmapSegment.create(filesFacade, initialPath, 0L, segmentSizeBytes); } try { - ringInProgress = new SegmentRing(initial, segmentSizeBytes); + if (!memoryMode) { + // Ordering is load-bearing for crash recovery: + // 1. initial segment durable (header + dirent), + // 2. manifest created (its own create fsyncs), + // 3. flag stamped on the segment. + // A crash after (1) recovers via the legacy + // (manifest-less) path; after (2) via the manifest + // path with an unflagged-but-valid empty active; + // after (3) it is the steady state. Every window + // self-heals without operator action. + initial.syncHeader(); + if (filesFacade.fsyncDir(sfDir) != 0) { + throw new MmapSegmentException( + "could not sync fresh SF segment directory " + sfDir); + } + initialManifest = SfManifest.create(filesFacade, sfDir, 0L, 0L); + initial.markManifestRequired(); + } + ringInProgress = new SegmentRing(initial, segmentSizeBytes, initialManifest); + initialManifest = null; } catch (Throwable t) { initial.close(); + if (initialManifest != null) { + initialManifest.close(); + SfManifest.removeFile(filesFacade, sfDir); + } if (initialPath != null) { - Files.remove(initialPath); + filesFacade.remove(initialPath); } throw t; } @@ -1247,17 +1280,24 @@ private static long segmentCleanupRank(String name) { } /** - * Unlinks every {@code .sfa} file under {@code dir}. Called only on - * clean shutdown when the ring confirms every published FSN has been - * acked — at that moment the slot has no recoverable work and the - * files are pure noise that would mislead the next sender's recovery. + * Unlinks every {@code .sfa} file under {@code dir}, then the SF + * manifest. Called only on clean shutdown when the ring confirms every + * published FSN has been acked — at that moment the slot has no + * recoverable work and the files are pure noise that would mislead the + * next sender's recovery. *

    - * Removal runs in ascending segment order and STOPS at the first - * failure, so whatever survives (a failure here, or a crash mid-loop) - * is always a contiguous top slice of the ring: recovery's - * FSN-contiguity check still passes, and the retained ack watermark - * (== the final acked FSN == the highest frame on disk) still covers - * every surviving frame, so a successor replays nothing. + * Crash safety comes from a single durable manifest update committed + * BEFORE the first unlink: collapsing the boundaries to + * {@code head == active} declares every frame below the active base + * trimmed, so any subset of surviving files recovers cleanly (stale + * files are discarded, a surviving active recovers as the whole chain, + * zero files with a leftover manifest is the recognized drain window + * and recovers as EMPTY). Removal still runs in ascending segment order + * and STOPS at the first failure; the retained ack watermark (== the + * final acked FSN) covers every surviving frame, so a successor replays + * nothing. Legacy slots without a manifest keep the pre-manifest + * argument: ascending-order removal preserves a contiguous top slice + * that passes FSN-contiguity. * * @return {@code true} only when enumeration succeeded and every * {@code .sfa} file was confirmed removed — the caller keeps the ack @@ -1300,13 +1340,54 @@ private static boolean unlinkAllSegmentFiles(FilesFacade filesFacade, String dir int byRank = Long.compare(segmentCleanupRank(a), segmentCleanupRank(b)); return byRank != 0 ? byRank : a.compareTo(b); }); - for (int i = 0, n = names.size(); i < n; i++) { - String path = dir + "/" + names.get(i); - if (!filesFacade.remove(path)) { - LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the " - + "residual files stay a contiguous, watermark-covered range", path); + // Manifest-aware drain protocol. Before removing anything, durably + // collapse the committed boundaries to head == active. From that + // moment EVERY crash state is a valid recovery input: + // - surviving data below the active base is "stale below head" + // (acked by definition here) and is discarded by recovery; + // - a surviving active-base segment recovers as the whole chain; + // - empty spares are validated extras; + // - zero segment files with the manifest still present is the + // recognized drain window and recovers as EMPTY. + // Without this barrier, deleting the file the manifest calls "head" + // would make the next startup fail on a slot that lost nothing. + // The manifest itself is removed last, after every segment unlink. + SfManifest manifest = null; + try { + try { + manifest = SfManifest.open(filesFacade, dir); + } catch (Throwable t) { + LOG.warn("close-time unlink could not read the SF manifest in {}; leaving " + + "all files for the next recovery", dir, t); return false; } + if (manifest != null && !names.isEmpty()) { + try { + long activeBase = manifest.activeBase(); + manifest.update(activeBase, activeBase); + } catch (Throwable t) { + LOG.warn("close-time unlink could not commit drain boundaries in {}; " + + "leaving all files for the next recovery", dir, t); + return false; + } + } + for (int i = 0, n = names.size(); i < n; i++) { + String path = dir + "/" + names.get(i); + if (!filesFacade.remove(path)) { + LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the " + + "residual files stay a watermark-covered, manifest-consistent range", path); + return false; + } + } + } finally { + if (manifest != null) { + manifest.close(); + } + } + if (!SfManifest.removeFile(filesFacade, dir)) { + // Safe to proceed: a manifest with zero segment files is the + // recognized drain window and recovers as EMPTY. + LOG.warn("could not remove SF manifest in {} after full segment cleanup", dir); } return true; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 226e047f..ff7a00da 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -63,10 +63,12 @@ public final class MmapSegment implements QuietCloseable { public static final int FILE_MAGIC = 0x31304653; // 'SF01' little-endian public static final int FRAME_HEADER_SIZE = 8; // u32 crc + u32 payloadLen public static final int HEADER_SIZE = 24; + public static final byte MANIFEST_REQUIRED_FLAG = 1; public static final byte VERSION = 1; private static final int[] CRC32C_TABLE = buildCrc32cTable(); private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class); + private final FilesFacade filesFacade; private final String path; private final long sizeBytes; // memoryBacked: true when the segment buffer lives in malloc'd native @@ -106,9 +108,10 @@ public final class MmapSegment implements QuietCloseable { // recovery callers for diagnostics. Final after construction. private final long tornTailBytes; - private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes, + private MmapSegment(FilesFacade filesFacade, String path, int fd, long mmapAddress, long sizeBytes, long baseSeq, long initialCursor, long frameCount, boolean memoryBacked, long tornTailBytes) { + this.filesFacade = filesFacade; this.path = path; this.fd = fd; this.mmapAddress = mmapAddress; @@ -149,9 +152,13 @@ public static MmapSegment create(String path, long baseSeq, long sizeBytes) { * that filesystem only. */ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes) { + return create(ff, path, baseSeq, sizeBytes, false); + } + + static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes, boolean manifestRequired) { long pathPtr = ff.allocNativePath(path); try { - return create(ff, pathPtr, path, baseSeq, sizeBytes); + return create(ff, pathPtr, path, baseSeq, sizeBytes, manifestRequired); } finally { ff.freeNativePath(pathPtr); } @@ -166,13 +173,17 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long * per-call {@code byte[]} + native-malloc the way the String overload does. */ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) { + return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, false); + } + + static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes, boolean manifestRequired) { if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) { throw new IllegalArgumentException( "sizeBytes too small for header + one minimal frame: " + sizeBytes); } - int fd = ff.openCleanRW(pathPtr); + int fd = ff.openRWExclusive(pathPtr); if (fd < 0) { - throw new MmapSegmentException("openCleanRW failed for " + displayPath); + throw new MmapSegmentException("exclusive create failed for " + displayPath); } // Reserve real disk blocks and advance EOF to sizeBytes in one // call. ENOSPC surfaces here, before the producer thread starts @@ -190,21 +201,21 @@ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPat } long addr = Files.FAILED_MMAP_ADDRESS; try { - addr = Files.mmap(fd, sizeBytes, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + addr = ff.mmap(fd, sizeBytes, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { throw new MmapSegmentException("mmap failed for " + displayPath); } // Header goes straight into the mapping — no separate write syscall. Unsafe.getUnsafe().putInt(addr, FILE_MAGIC); Unsafe.getUnsafe().putByte(addr + 4, VERSION); - Unsafe.getUnsafe().putByte(addr + 5, (byte) 0); // flags + Unsafe.getUnsafe().putByte(addr + 5, manifestRequired ? MANIFEST_REQUIRED_FLAG : (byte) 0); // flags Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved Unsafe.getUnsafe().putLong(addr + 8, baseSeq); Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros()); - return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L); + return new MmapSegment(ff, displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L); } catch (Throwable t) { if (addr != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT); + ff.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT); } ff.close(fd); // mmap (or header writes) failed after a successful allocate — @@ -238,7 +249,7 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) { Unsafe.getUnsafe().putShort(addr + 6, (short) 0); Unsafe.getUnsafe().putLong(addr + 8, baseSeq); Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros()); - return new MmapSegment(null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L); + return new MmapSegment(null, null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L); } catch (Throwable t) { Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT); throw t; @@ -284,7 +295,11 @@ public static MmapSegment openExisting(String path) { public static MmapSegment openExisting(FilesFacade ff, String path) { long fileSize = ff.length(path); if (fileSize < HEADER_SIZE) { - throw new MmapSegmentException("file shorter than header: " + path + " size=" + fileSize); + // Corruption, not an operational error: the bytes themselves prove + // this cannot be a whole segment (a create() is never durable at a + // sub-header size — allocate() reserves the full extent up front). + throw new MmapSegmentCorruptionException( + "file shorter than header: " + path + " size=" + fileSize); } int fd = ff.openRW(path); if (fd < 0) { @@ -292,17 +307,22 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { } long addr = Files.FAILED_MMAP_ADDRESS; try { - addr = Files.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + addr = ff.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { throw new MmapSegmentException("mmap failed for " + path); } int magic = Unsafe.getUnsafe().getInt(addr); if (magic != FILE_MAGIC) { - throw new MmapSegmentException( + throw new MmapSegmentCorruptionException( "bad magic in " + path + ": 0x" + Integer.toHexString(magic)); } byte version = Unsafe.getUnsafe().getByte(addr + 4); if (version != VERSION) { + // Deliberately NOT the corruption subtype: an unsupported + // version is a well-formed segment written by a different + // client build (e.g. after a downgrade). Quarantine-renaming + // it would strand its frames for the writer that CAN read it; + // failing recovery keeps the slot intact for that writer. throw new MmapSegmentException("unsupported version in " + path + ": " + version); } long baseSeq = Unsafe.getUnsafe().getLong(addr + 8); @@ -314,7 +334,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { // checks (which would place the segment last in baseSeq order // and trip the FSN-gap throw, taking the whole recovery down). if (baseSeq < 0L) { - throw new MmapSegmentException( + throw new MmapSegmentCorruptionException( "bad baseSeq in " + path + ": " + baseSeq); } long lastGood = scanFrames(addr, fileSize); @@ -328,10 +348,10 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { + "Investigate disk health or unexpected writer crash.", path, tornTail, lastGood, fileSize, count); } - return new MmapSegment(path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail); + return new MmapSegment(ff, path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail); } catch (Throwable t) { if (addr != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT); + ff.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT); } ff.close(fd); // The header reads above (magic/version/baseSeq) run before @@ -347,7 +367,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { // covers the header block and any future reader placed ahead of the // scan. if (isMmapAccessFault(t)) { - throw new MmapSegmentException( + throw new MmapSegmentCorruptionException( "unreadable mapped header page in " + path + " (unbacked/sparse page 0): " + t.getMessage(), t); } @@ -379,16 +399,37 @@ public void close() { if (memoryBacked) { Unsafe.free(mmapAddress, sizeBytes, MemoryTag.NATIVE_DEFAULT); } else { - Files.munmap(mmapAddress, sizeBytes, MemoryTag.MMAP_DEFAULT); + filesFacade.munmap(mmapAddress, sizeBytes, MemoryTag.MMAP_DEFAULT); } mmapAddress = 0; } if (fd >= 0) { - Files.close(fd); + filesFacade.close(fd); fd = -1; } } + boolean manifestRequired() { + return !memoryBacked && (Unsafe.getUnsafe().getByte(mmapAddress + 5) & MANIFEST_REQUIRED_FLAG) != 0; + } + + void markManifestRequired() { + if (memoryBacked || manifestRequired()) { + return; + } + Unsafe.getUnsafe().putByte(mmapAddress + 5, MANIFEST_REQUIRED_FLAG); + syncHeader(); + } + + void syncHeader() { + if (memoryBacked) { + return; + } + if (filesFacade.msync(mmapAddress, HEADER_SIZE, false) != 0 || filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync segment header " + path); + } + } + public boolean isFull() { return capacityRemaining() <= 0; } @@ -402,7 +443,7 @@ public void msync() { if (memoryBacked) return; // no on-disk pages to flush long pub = publishedCursor; if (pub > HEADER_SIZE) { - Files.msync(mmapAddress, pub, false); + filesFacade.msync(mmapAddress, pub, false); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java new file mode 100644 index 00000000..34a89f38 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * Positively-identified segment corruption: the file's own bytes prove it is + * not (or no longer) a readable SF segment — truncated below the fixed header, + * wrong magic, a negative {@code baseSeq}, or an unreadable (unbacked/torn) + * header page. Distinct from its parent {@link MmapSegmentException}, which + * recovery treats as an operational failure (open/mmap error on a file + * whose contents may be perfectly intact) and must therefore be fatal. + *

    + * Recovery quarantines corruption (rename to {@code .corrupt}) and + * relies on manifest boundaries / FSN contiguity to decide whether the + * quarantined file was load-bearing; operational failures always abort + * startup so a transient {@code EMFILE}/{@code ENOMEM} can never silently + * drop durable frames. + */ +public final class MmapSegmentCorruptionException extends MmapSegmentException { + + public MmapSegmentCorruptionException(String message) { + super(message); + } + + public MmapSegmentCorruptionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java index eec0c0d9..e54eecb5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java @@ -29,8 +29,14 @@ * too short for header, etc. Indicates the segment is unusable, not that * the disk is full (the latter surfaces as backpressure on the producer * via {@link io.questdb.client.cutlass.qwp.client.LineSenderException}). + *

    + * Recovery distinguishes two flavors: this base type marks operational + * failures (open/mmap/enumeration errors — the file's contents may be fine, + * so recovery must fail closed), while the {@link MmapSegmentCorruptionException} + * subtype marks positively-identified corruption in the file's own + * bytes, which recovery may quarantine instead of aborting. */ -public final class MmapSegmentException extends RuntimeException { +public class MmapSegmentException extends RuntimeException { public MmapSegmentException(String message) { super(message); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 6bb617ad..5319ee65 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -808,12 +808,19 @@ private boolean serviceRing0(RingEntry e) { // rotation. spare = MmapSegment.create(filesFacade, pathScratch.ptr(), path, - e.ring.nextSeqHint(), segmentSizeBytes); + e.ring.nextSeqHint(), segmentSizeBytes, true); } Runnable installHook = beforeInstallSyncHook; if (installHook != null) { installHook.run(); } + if (!memoryMode) { + spare.syncHeader(); + if (filesFacade.fsyncDir(e.dir) != 0) { + throw new MmapSegmentException( + "could not sync hot-spare directory " + e.dir); + } + } // Install + commit atomically under the manager lock. // If `e.ring` was deregistered between the snapshot // above and now, abandoning the spare here is the only @@ -844,14 +851,15 @@ private boolean serviceRing0(RingEntry e) { } catch (Throwable ignored) { } } - // Remove the file even when spare is null (i.e. when - // MmapSegment.create itself threw): MmapSegment.create's - // catch already best-effort removes, but if anything - // before mmap (e.g. an exception thrown by the JVM - // between openCleanRW and the try block) leaves a file - // on disk, this is the second-line defense. Repeated - // unlink on an already-removed path is a harmless no-op. - if (path != null) { + // Only remove the file when the spare object exists, i.e. + // MmapSegment.create succeeded and ownership is ours but + // installation was rejected (ring deregistered/closed). + // When create() itself threw, its catch already removed + // anything it put on disk -- and with exclusive create + // (O_EXCL) a failure can also mean the path was ALREADY + // occupied by a file some other lifecycle owns, which a + // blanket unlink here would destroy. + if (path != null && spare != null) { filesFacade.remove(path); } } @@ -964,6 +972,13 @@ private boolean serviceRing0(RingEntry e) { MmapSegment next = e.ring.nextSealedAfter(segment); String path = segment.path(); try { + // Durably commit the head advance past this segment before + // unlinking it. The ring recomputes the successor under its + // own monitor -- computing it here from `next` would race + // with a concurrent rotation sealing the active, letting the + // head leapfrog a still-unacked sealed segment (whose file a + // later recovery would then discard as "stale below head"). + e.ring.advanceManifestHeadPast(segment); segment.close(); // A retry after a post-unlink directory-sync failure sees an // already absent path. Treat absence as the prior successful diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 8f5fddbb..3ad376f3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; import org.jetbrains.annotations.TestOnly; @@ -77,6 +78,7 @@ public final class SegmentRing implements QuietCloseable { // Test-only operation count for deterministic trim-complexity assertions. private static long trimMovedReferences; private final long maxBytesPerSegment; + private final SfManifest manifest; // Sealed segments in baseSeq order, oldest first. Active is held separately. // Single-writer (producer thread, on rotation); single-reader at trim time // (the segment manager). For now, both sides synchronize via the single- @@ -129,10 +131,15 @@ public final class SegmentRing implements QuietCloseable { * frameCount == 0); typically supplied by the segment manager at startup. */ public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) { + this(initialActive, maxBytesPerSegment, null); + } + + SegmentRing(MmapSegment initialActive, long maxBytesPerSegment, SfManifest manifest) { if (initialActive == null) { throw new IllegalArgumentException("initialActive must not be null"); } this.active = initialActive; + this.manifest = manifest; this.maxBytesPerSegment = maxBytesPerSegment; // 3/4 of capacity gives the manager a full quarter-segment of producer // runway before backpressure kicks in. Long math, no float, no alloc. @@ -142,195 +149,488 @@ public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) { } /** - * Recovers a ring from segments already on disk in {@code sfDir}. Used at - * sender startup when the user's previous session left durable but - * not-yet-acked frames behind. Walks every {@code *.sfa} file in the - * directory, opens each via {@link MmapSegment#openExisting}, and - * arranges them by baseSeq: - *

      - *
    • Highest-baseSeq segment becomes the active (further appends land - * there until it fills, at which point normal rotation kicks in).
      • - *
    • All others become sealed segments awaiting ACK and trim.
      • - *
    - * Returns {@code null} if the directory is empty or contains no - * recognizable {@code .sfa} files -- the caller should then construct a - * fresh ring with {@link #SegmentRing(MmapSegment, long)} and a freshly - * created initial segment. - *

    - * Recovery is best-effort: a single bad-magic file is silently skipped - * (logged-then-ignored is the right call here; a stray unrelated file in - * the SF dir shouldn't take the whole sender down). A failure to open - * an otherwise-valid segment IS fatal -- the caller's data integrity - * depends on every segment being readable. + * Compatibility wrapper using the production facade. New startup code uses + * {@link #recover(FilesFacade, String, long)} so EMPTY is explicit. */ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) { - if (!Files.exists(sfDir)) { - return null; + return openExisting(FilesFacade.INSTANCE, sfDir, maxBytesPerSegment); + } + + /** + * Facade-aware variant of {@link #openExisting(String, long)}: every + * filesystem touch (enumeration, open, mmap, quarantine rename, manifest + * I/O) goes through {@code filesFacade} so recovery's fail-closed behavior + * can be fault-injected in tests. Returns {@code null} when the slot holds + * nothing recoverable; throws {@link MmapSegmentException} when the slot's + * state cannot be proven safe. + */ + public static SegmentRing openExisting(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { + Recovery recovery = recover(filesFacade, sfDir, maxBytesPerSegment); + return recovery.status == RecoveryStatus.EMPTY ? null : recovery.ring; + } + + /** + * Exhaustively discovers and validates an SF slot without mutation. Only + * after enumeration, opens, CRC scans, contiguity and manifest boundaries + * all succeed does it migrate a legacy chain or discard validated spares. + */ + static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { + if (!filesFacade.exists(sfDir)) { + return Recovery.empty(); } - ObjList opened = new ObjList<>(); - long find = Files.findFirst(sfDir); + ObjList names = new ObjList<>(); + long find = filesFacade.findFirst(sfDir); if (find < 0) { - LOG.warn("openExisting could not enumerate {} - treating as empty, " - + "but this may indicate a permission or transient error", sfDir); - return null; - } - if (find == 0) { - return null; + throw new MmapSegmentException("could not enumerate SF directory " + sfDir); } - // Outer try-catch: anything escaping the recovery body -- IOOBE from - // ObjList growth, OOM from native mmap during MmapSegment.openExisting, - // unforeseen RuntimeException from the contiguity check, etc. -- must - // not leave fds + mmaps owned by `opened` orphaned. Close every - // recovered segment and rethrow so the engine surfaces the failure. - try { + if (find > 0) { + int rc = 1; try { - int rc = 1; while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); + String name = Files.utf8ToString(filesFacade.findName(find)); if (name != null && name.endsWith(".sfa")) { - String path = sfDir + "/" + name; - MmapSegment seg = null; - try { - seg = MmapSegment.openExisting(path); - // Filter out empty leftovers -- typically hot-spare - // segments the manager pre-allocated for a prior - // session that never got rotated into active. They - // carry the provisional baseSeq=0 and frameCount=0, - // which would otherwise collide with the real - // baseSeq=0 segment and trip the contiguity check - // below. No data to recover; close and unlink. - // Without the unlink the file persists across crash - // cycles and the disk leak compounds with every - // unclean shutdown. - // - // CAUTION: only unlink when the file is genuinely - // empty past the header. If frame[0] failed CRC - // (bit-rot, partial-page-write at crash, etc.) but - // valid frames followed, scanFrames returns - // lastGood=HEADER_SIZE and frameCount=0 -- yet - // tornTailBytes is non-zero. Treating that as - // "empty hot-spare" would silently destroy every - // surviving frame. Quarantine to .corrupt - // instead so a postmortem can recover what's left. - if (seg.frameCount() == 0) { - long torn = seg.tornTailBytes(); - seg.close(); - seg = null; - if (torn > 0) { - Files.rename(path, path + ".corrupt"); - } else { - Files.remove(path); - } - } else { - opened.add(seg); - seg = null; - } - } catch (MmapSegmentException t) { - // Per-file data error (bad magic, bad header, - // unsupported version, mmap rejection on this one - // file). Don't take down recovery for one corrupt - // .sfa -- log and skip so siblings still recover. - // Resource exhaustion (OOM) and programmer errors - // (IOOBE) deliberately propagate to the outer - // catch, which closes every already-recovered - // segment and rethrows: continuing the loop after - // an OOM would just fail again on the next file - // while silently leaking the segments we managed - // to recover before it. - LOG.warn("openExisting: skipping {} -- {}", path, t.toString()); - } finally { - // Close any seg whose ownership wasn't transferred - // (either to opened, or via the empty-branch close - // above). Fires on a propagating throw between - // open and transfer -- most importantly an OOM - // from ObjList.add growing its backing array - // after the mmap+fd were already acquired. - if (seg != null) { - try { - seg.close(); - } catch (Throwable closeErr) { - LOG.warn("openExisting: error closing in-flight segment {}", - path, closeErr); - } - } - } + names.add(name); } - rc = Files.findNext(find); + rc = filesFacade.findNext(find); + } + if (rc < 0) { + throw new MmapSegmentException("could not fully enumerate SF directory " + sfDir); } } finally { - Files.findClose(find); + filesFacade.findClose(find); } - if (opened.size() == 0) { - return null; + } + + ObjList all = new ObjList<>(); + // Files whose own bytes prove corruption (bad magic, sub-header size, + // negative baseSeq, unreadable header page). They are excluded from + // the chain and quarantined to .corrupt — but only AFTER the + // surviving chain validates (or resolves to EMPTY), so a failed + // recovery never mutates the slot. Whether a quarantined file was + // load-bearing is decided by the manifest-boundary / contiguity + // checks below, not by the skip itself. Operational open errors + // (EMFILE, EACCES, mmap rejection, unsupported version) are NOT in + // this bucket: they throw the plain MmapSegmentException type and + // abort recovery, because the underlying file may be perfectly + // intact and silently dropping it could lose durable frames. + ObjList corruptPaths = null; + SfManifest manifest = null; + try { + for (int i = 0, n = names.size(); i < n; i++) { + String path = sfDir + "/" + names.get(i); + try { + all.add(MmapSegment.openExisting(filesFacade, path)); + } catch (MmapSegmentCorruptionException e) { + LOG.warn("recovery: {} is not a readable SF segment; excluding it and " + + "deferring quarantine until the surviving chain validates -- {}", + path, e.toString()); + if (corruptPaths == null) { + corruptPaths = new ObjList<>(); + } + corruptPaths.add(path); + } catch (MmapSegmentException e) { + throw new MmapSegmentException("recovery failed for recognized segment " + path, e); + } } - // Sort by baseSeq ascending. Worst-case segment count is - // sf_max_total_bytes / sf_max_bytes -- at the documented ceiling - // (1 TiB / 64 MiB) that is ~16K entries, where an O(N²) sort spends - // multiple seconds in compares + shifts before the I/O thread can - // start. In-place quicksort with median-of-three pivot keeps the - // no-allocation discipline of the surrounding code; median-of-three - // is required because readdir on many filesystems returns entries - // in lexicographic (== baseSeq-hex) order and a naive first-element - // pivot would degrade back to O(N²) on exactly that common case. - sortByBaseSeq(opened, 0, opened.size()); - // Sanity: the recovered segments must form a contiguous FSN range. - // Detect gaps so they don't silently produce duplicate or missing - // FSNs after recovery. A gap means a segment went missing (a - // manual deletion) or a sealed segment under-recovered -- its tail - // was cut short by a sparse/unbacked page or a mid-file media error - // (bad sector), the same class of fault scanFrames tolerates on the - // active segment but which corrupts the range on a sealed one. - for (int i = 1, n = opened.size(); i < n; i++) { - MmapSegment prev = opened.get(i - 1); - MmapSegment curr = opened.get(i); - long expected = prev.baseSeq() + prev.frameCount(); - if (curr.baseSeq() != expected) { - throw new MmapSegmentException( - "FSN gap in recovered segments: prev baseSeq=" + prev.baseSeq() - + " frameCount=" + prev.frameCount() - + " expected next baseSeq=" + expected - + " but got " + curr.baseSeq() - + " -- a segment was deleted, or a sealed segment's tail was" - + " truncated (sparse/unbacked page or disk media error);" - + " check disk health"); + manifest = SfManifest.open(filesFacade, sfDir); + if (all.size() == 0) { + if (corruptPaths != null) { + // Nothing valid survived. With a manifest this is still a + // hole we can prove (boundaries reference segments that are + // now unreadable) -- fail without mutating. Without one, + // legacy semantics apply: quarantine and start fresh. + if (manifest != null) { + throw new MmapSegmentException("every SF segment in " + sfDir + + " is corrupt but " + SfManifest.FILE_NAME + + " references durable data"); + } + quarantineCorrupt(filesFacade, corruptPaths); + return Recovery.empty(); } + if (manifest != null) { + // No .sfa files at all. Two legitimate protocols produce + // this: the close-time drain unlinks the last segment + // before it removes the manifest, and a fresh-start crash + // can leave a boundary-less (0,0) manifest behind. In + // both cases nothing recoverable exists, so accept EMPTY + // -- but shout, because a manual wipe of segment files + // looks identical and the operator should know. + LOG.warn("SF manifest exists in {} with no segment files " + + "(clean-drain or fresh-start crash window, or manual " + + "segment removal); discarding it and starting fresh", sfDir); + manifest.close(); + manifest = null; + if (!SfManifest.removeFile(filesFacade, sfDir)) { + throw new MmapSegmentException( + "could not remove stale SF manifest in " + sfDir); + } + } + return Recovery.empty(); } - // Publish the immutable in-memory successor chain before exposing - // the recovered ring. The final sealed segment links to the active. - for (int i = 1, n = opened.size(); i < n; i++) { - opened.get(i - 1).linkSuccessor(opened.get(i)); + + ObjList data = new ObjList<>(); + boolean requiresManifest = false; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + requiresManifest |= segment.manifestRequired(); + if (segment.frameCount() > 0) { + data.add(segment); + } } - // The newest segment becomes the active. Even if it's full, that's OK: - // the next appendOrFsn returns BACKPRESSURE_NO_SPARE, the manager - // installs a hot spare, the producer rotates. Same fast path as a - // mid-life ring. - int last = opened.size() - 1; - MmapSegment active = opened.get(last); - opened.remove(last); - SegmentRing ring = new SegmentRing(active, maxBytesPerSegment); - // Older segments become sealed in baseSeq order. - for (int i = 0, n = opened.size(); i < n; i++) { - ring.sealedSegments.add(opened.get(i)); + sortByBaseSeq(data, 0, data.size()); + if (manifest == null && requiresManifest) { + throw new MmapSegmentException("new-format SF segment exists but " + + SfManifest.FILE_NAME + " is missing"); } - return ring; + + MmapSegment active; + ObjList chain = new ObjList<>(); + long headBase; + long activeBase; + if (manifest == null) { + if (data.size() > 0) { + validateContiguous(data); + for (int i = 0, n = data.size(); i < n; i++) { + chain.add(data.get(i)); + } + active = chain.get(chain.size() - 1); + headBase = chain.get(0).baseSeq(); + activeBase = active.baseSeq(); + } else { + active = chooseEmptyInitial(all, sfDir); + if (active == null) { + // Legacy slot holding only empty leftovers, every one + // of them torn (a clean empty would have been chosen). + // Nothing recoverable: quarantine the torn evidence, + // drop the clean debris, start fresh -- exactly the + // pre-manifest behavior. + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove empty SF leftover {}", path); + } + } + all.clear(); + quarantineCorrupt(filesFacade, corruptPaths); + return Recovery.empty(); + } + chain.add(active); + headBase = active.baseSeq(); + activeBase = active.baseSeq(); + } + manifest = SfManifest.create(filesFacade, sfDir, headBase, activeBase); + for (int i = 0, n = chain.size(); i < n; i++) { + chain.get(i).markManifestRequired(); + } + } else { + headBase = manifest.headBase(); + activeBase = manifest.activeBase(); + for (int i = 0, n = data.size(); i < n; i++) { + MmapSegment segment = data.get(i); + long end = segment.baseSeq() + segment.frameCount(); + if (segment.baseSeq() < headBase) { + if (end > headBase) { + throw new MmapSegmentException("segment overlaps committed SF head boundary"); + } + continue; // acknowledged stale file after manifest-before-unlink crash + } + if (segment.baseSeq() > activeBase) { + throw new MmapSegmentException("segment exists beyond committed SF active boundary"); + } + chain.add(segment); + } + if (chain.size() > 0) { + validateContiguous(chain); + if (chain.get(0).baseSeq() != headBase) { + throw new MmapSegmentException("missing expected SF head segment at base " + headBase); + } + } + active = findActive(all, activeBase); + if (active == null) { + if (chain.size() == 0 && headBase == activeBase && corruptPaths == null) { + // Clean-drain crash window: the close-time drain first + // durably collapses the boundaries to head == active + // (declaring every frame acked), then unlinks segments + // in ascending order -- so dying between the active's + // unlink and the spare's/manifest's leaves exactly + // this state: no data frame anywhere, no file at the + // committed active base, only empty spares and/or + // acked stale files. Nothing recoverable exists; + // accept EMPTY and clear the debris. Guarded on + // corruptPaths because an unreadable .sfa of unknown + // identity could be the real active -- in that case + // fail closed instead of guessing. + LOG.warn("SF manifest in {} has collapsed boundaries ({}) with no " + + "segment at the active base and no recovered frames; " + + "accepting the clean-drain crash window as empty", + sfDir, activeBase); + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove drained SF leftover {}", path); + } + } + all.clear(); + manifest.close(); + manifest = null; + if (!SfManifest.removeFile(filesFacade, sfDir)) { + throw new MmapSegmentException( + "could not remove stale SF manifest in " + sfDir); + } + return Recovery.empty(); + } + throw new MmapSegmentException("missing expected SF active segment at base " + activeBase); + } + if (chain.size() == 0) { + if (headBase != activeBase || active.frameCount() != 0 || corruptPaths != null) { + // corruptPaths guard: with an unreadable .sfa in the + // slot, the innocent-looking empty at the active base + // could be a leftover spare coincidentally carrying + // the same provisional baseSeq as a corrupted real + // active -- accepting it would quarantine unacked + // frames and re-issue their FSNs. Fail closed. + throw new MmapSegmentException( + "missing SF chain between committed boundaries" + + (corruptPaths != null + ? " (a corrupt segment prevents proving the empty state)" : "")); + } + chain.add(active); + } else if (chain.get(chain.size() - 1) != active) { + MmapSegment tail = chain.get(chain.size() - 1); + long chainEnd = tail.baseSeq() + tail.frameCount(); + if (corruptPaths == null && active.frameCount() == 0 && active.baseSeq() == chainEnd) { + // Rotation committed (manifest fsync'd, promoted spare's + // header synced) but the process/OS died before a single + // frame of the new active reached disk: the sealed chain + // ends exactly where the empty active begins. Legal + // crash state -- resume appending into it. Refused when + // corrupt segments exist (same stand-in hazard as the + // empty-chain acceptance above). + chain.add(active); + } else { + throw new MmapSegmentException( + "missing expected SF active/tail segment at base " + activeBase); + } + } + for (int i = 0, n = chain.size() - 1; i < n; i++) { + if (chain.get(i).tornTailBytes() > 0) { + throw new MmapSegmentException("corrupt torn tail in sealed SF segment " + chain.get(i).path()); + } + } + for (int i = 0, n = chain.size(); i < n; i++) { + chain.get(i).markManifestRequired(); + } + } + + for (int i = 1, n = chain.size(); i < n; i++) { + chain.get(i - 1).linkSuccessor(chain.get(i)); + } + SegmentRing ring = new SegmentRing(active, maxBytesPerSegment, manifest); + manifest = null; + for (int i = 0, n = chain.size() - 1; i < n; i++) { + ring.sealedSegments.add(chain.get(i)); + } + // Ownership of the chain transferred. Clean up only validated + // extras; recovery is already successful, so cleanup failure + // must never turn startup into a partially-mutating error or + // orphan the constructed ring -- swallow and let the next + // startup re-examine the leftovers. Extras with a torn tail + // carry evidence of attempted writes -- keep the bytes under a + // .corrupt name instead of unlinking them. + try { + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (!containsIdentity(chain, segment)) { + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove validated stale/empty SF segment {}", path); + } + } + } + all.clear(); + quarantineCorrupt(filesFacade, corruptPaths); + } catch (Throwable cleanupError) { + LOG.warn("post-recovery cleanup failed; leftover files will be " + + "re-examined on the next startup", cleanupError); + } + return Recovery.recovered(ring); } catch (Throwable t) { - // Close every recovered MmapSegment that's still in `opened`. - // After the success path, `opened` no longer contains the active - // segment (removed above), but the sealed segments transferred to - // ring.sealedSegments are still owned by the ring once it's - // returned -- so this catch only fires before the return statement. - for (int i = 0, n = opened.size(); i < n; i++) { + for (int i = 0, n = all.size(); i < n; i++) { try { - opened.get(i).close(); - } catch (Throwable closeErr) { - LOG.warn("openExisting: error closing recovered segment during cleanup", - closeErr); + all.get(i).close(); + } catch (Throwable closeError) { + LOG.warn("error closing SF segment after recovery failure", closeError); } } + if (manifest != null) { + manifest.close(); + } throw t; } } + /** + * Durably advances the manifest head past {@code trimming} (the sealed + * segment the manager is about to unlink). The successor and the current + * active are both read under the ring monitor, so a concurrent rotation + * (which also mutates the manifest under this monitor) can never make the + * head leapfrog a still-live sealed segment: if rotation sealed the old + * active after the caller's snapshot, {@code trimming.successor()} now + * points at that sealed segment, not at the new active. + */ + synchronized void advanceManifestHeadPast(MmapSegment trimming) { + if (manifest == null) { + return; + } + MmapSegment successor = trimming.successor(); + long newHeadBase = (successor == null || successor == active) + ? active.baseSeq() + : successor.baseSeq(); + manifest.update(newHeadBase, active.baseSeq()); + } + + /** + * Picks the clean (untorn) empty segment to reuse as a legacy slot's + * initial active, preferring {@code sf-initial.sfa}. Returns {@code null} + * when no clean empty exists; torn empties are never reused here because + * their bytes are quarantine evidence, not blank space. + */ + private static MmapSegment chooseEmptyInitial(ObjList all, String sfDir) { + String initialPath = sfDir + "/sf-initial.sfa"; + MmapSegment selected = null; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment.frameCount() != 0 || segment.tornTailBytes() > 0) { + continue; + } + if (selected == null || initialPath.equals(segment.path())) { + selected = segment; + } + } + return selected; + } + + /** Renames every collected corrupt path to {@code .corrupt}, best-effort. */ + private static void quarantineCorrupt(FilesFacade filesFacade, ObjList corruptPaths) { + if (corruptPaths == null) { + return; + } + for (int i = 0, n = corruptPaths.size(); i < n; i++) { + quarantineFile(filesFacade, corruptPaths.get(i)); + } + } + + private static void quarantineFile(FilesFacade filesFacade, String path) { + if (filesFacade.rename(path, path + ".corrupt") != 0) { + LOG.warn("could not quarantine {} to {}.corrupt; it will be re-examined " + + "on the next recovery", path, path); + } + } + + private static boolean containsIdentity(ObjList list, MmapSegment value) { + for (int i = 0, n = list.size(); i < n; i++) { + if (list.get(i) == value) return true; + } + return false; + } + + /** + * Locates the segment the manifest's {@code activeBase} refers to. + * Preference order among same-base candidates: + *

      + *
    1. a segment with recovered frames (the durable chain tail);
      2. + *
    2. an empty segment with a torn tail (the promoted active whose + * first frame write was cut short — an attempted write marks it as + * the one rotation actually exposed);
      3. + *
    3. a clean empty segment.
      4. + *
    + * Multiple equivalent empties at the same base are NOT an error: a fresh + * start or a rotation crash routinely leaves both the initial/promoted + * segment and a provisioned hot spare carrying the same provisional + * baseSeq. They are interchangeable blanks — pick one deterministically + * and let the extras cleanup discard the rest. Bricking startup on this + * state would turn every "kill -9 shortly after start" into a manual + * repair. + */ + private static MmapSegment findActive(ObjList all, long activeBase) { + MmapSegment tornEmpty = null; + MmapSegment cleanEmpty = null; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment.baseSeq() != activeBase) { + continue; + } + if (segment.frameCount() > 0) { + return segment; + } + if (segment.tornTailBytes() > 0) { + if (tornEmpty == null) { + tornEmpty = segment; + } + } else if (cleanEmpty == null) { + cleanEmpty = segment; + } + } + return tornEmpty != null ? tornEmpty : cleanEmpty; + } + + private static void validateContiguous(ObjList segments) { + for (int i = 1, n = segments.size(); i < n; i++) { + MmapSegment previous = segments.get(i - 1); + MmapSegment current = segments.get(i); + long expected = previous.baseSeq() + previous.frameCount(); + if (current.baseSeq() != expected) { + throw new MmapSegmentException("FSN gap in recovered segments: expected " + + expected + " but got " + current.baseSeq()); + } + } + } + + static final class Recovery { + private final SegmentRing ring; + private final RecoveryStatus status; + + private Recovery(RecoveryStatus status, SegmentRing ring) { + this.status = status; + this.ring = ring; + } + + static Recovery empty() { + return new Recovery(RecoveryStatus.EMPTY, null); + } + + SegmentRing ring() { + return ring; + } + + static Recovery recovered(SegmentRing ring) { + return new Recovery(RecoveryStatus.RECOVERED, ring); + } + + RecoveryStatus status() { + return status; + } + } + + enum RecoveryStatus { + EMPTY, + RECOVERED + } + /** * Highest FSN that the server has ACK'd. Read by the segment manager to * decide which sealed segments are safe to munmap + unlink. @@ -396,11 +696,45 @@ public long appendOrFsn(long payloadAddr, int payloadLen) { MmapSegment previous = active; long actualBase = previous.baseSeq() + previous.frameCount(); spare.rebaseSeq(actualBase); + if (manifest != null) { + // Make the spare's rebased identity durable BEFORE the manifest + // references it. Without this barrier an OS crash could leave a + // durable manifest pointing at baseSeq=actualBase while the + // spare's on-disk header still carries the manager's + // provisional guess -- recovery would then find no segment at + // the committed active boundary and fail a startup that lost + // nothing. One msync per rotation, amortized over a whole + // segment of appends; runs outside the monitor because the + // spare is not yet visible to any other thread. + // + // Deliberately NOT msync'd here: the sealed predecessor's + // data pages. A power loss can therefore tear the sealed + // tail after the boundary is committed, and recovery will + // fail closed on chainEnd != activeBase. That is the + // intended semantics -- page-level durability of frame data + // follows the sender's opt-in msync cadence, and recovery + // must refuse to guess when the two disagree. + spare.syncHeader(); + } // Publish the successor before the volatile active promotion. The // same monitor protects the sealed list and nextSealedAfter's trim // fallback, while the volatile link also remains readable from a // current segment after the manager removes and closes it. synchronized (this) { + if (manifest != null) { + // Inside the monitor: serialized with the trim path's + // advanceManifestHeadPast so neither writer publishes a + // boundary computed from a state the other has already + // moved past (SfManifest additionally clamps monotonic). + // BEFORE any ring mutation: if the manifest fsync throws, + // the rotation never happened -- previous stays active, + // the spare stays installed, and the producer's retry + // re-runs this block from a consistent state. + long headBase = sealedHead < sealedSegments.size() + ? sealedSegments.get(sealedHead).baseSeq() + : previous.baseSeq(); + manifest.update(headBase, actualBase); + } previous.linkSuccessor(spare); sealedSegments.add(previous); active = spare; @@ -461,6 +795,9 @@ public synchronized void close() { } sealedSegments.clear(); sealedHead = 0; + if (manifest != null) { + manifest.close(); + } } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java new file mode 100644 index 00000000..f09ce94a --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java @@ -0,0 +1,282 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.std.Crc32c; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Crash-safe boundary record for an SF segment chain. Two fixed-size, + * independently CRC-protected records alternate on update. Recovery selects + * the valid record with the greatest generation, so a torn update cannot + * erase the previous committed head/active boundary. + */ +final class SfManifest implements QuietCloseable { + static final String FILE_NAME = "sf-manifest.bin"; + private static final Logger LOG = LoggerFactory.getLogger(SfManifest.class); + private static final int CRC_OFFSET = 60; + private static final long FILE_SIZE = 128; + private static final int MAGIC = 0x314d4653; // SFM1 little-endian + private static final int RECORD_SIZE = 64; + private static final int VERSION = 1; + private final int fd; + private final FilesFacade filesFacade; + private final String path; + private long activeBase; + private boolean closed; + private long generation; + private long headBase; + + private SfManifest(FilesFacade filesFacade, String path, int fd, + long generation, long headBase, long activeBase) { + this.filesFacade = filesFacade; + this.path = path; + this.fd = fd; + this.generation = generation; + this.headBase = headBase; + this.activeBase = activeBase; + } + + static SfManifest create(FilesFacade filesFacade, String dir, long headBase, long activeBase) { + String path = dir + "/" + FILE_NAME; + int fd = filesFacade.openRWExclusive(path); + if (fd < 0) { + throw new MmapSegmentException("exclusive create failed for SF manifest " + path); + } + boolean success = false; + try { + if (!filesFacade.allocate(fd, FILE_SIZE)) { + throw new MmapSegmentException("could not allocate SF manifest " + path); + } + SfManifest manifest = new SfManifest(filesFacade, path, fd, 0, -1, -1); + manifest.update(headBase, activeBase); + if (filesFacade.fsyncDir(dir) != 0) { + throw new MmapSegmentException("could not sync SF manifest directory " + dir); + } + success = true; + return manifest; + } finally { + if (!success) { + filesFacade.close(fd); + filesFacade.remove(path); + } + } + } + + static SfManifest open(FilesFacade filesFacade, String dir) { + String path = dir + "/" + FILE_NAME; + if (!filesFacade.exists(path)) { + return null; + } + if (filesFacade.length(path) != FILE_SIZE) { + // A wrong-sized manifest is creation debris: create() reaches the + // full FILE_SIZE via allocate() before writing the first record, + // so a mis-sized file proves the crash happened before any + // boundary was ever committed — nothing can depend on it yet + // (segment flags are stamped only after create() returns). Treat + // as absent so startup self-heals; genuine post-creation loss is + // still caught by the manifest-required flag check. + quarantineDebris(filesFacade, path, "wrong size " + filesFacade.length(path)); + return null; + } + int fd = filesFacade.openRW(path); + if (fd < 0) { + throw new MmapSegmentException("could not open SF manifest " + path); + } + long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Record first = readRecord(filesFacade, fd, buffer, 0); + Record second = readRecord(filesFacade, fd, buffer, RECORD_SIZE); + Record selected; + if (first == null) { + selected = second; + } else if (second == null || first.generation > second.generation) { + selected = first; + } else { + selected = second; + } + if (selected == null) { + // No valid record in either slot. create() makes the first + // record durable (write + fsync) before returning, and every + // later update() rewrites only ONE slot — so a torn update + // leaves the sibling record intact. Zero valid records + // therefore proves a creation crash, not boundary loss. + // Self-heal by treating the file as absent. If durable state + // DID depend on a manifest (flags stamped, i.e. double-slot + // bit rot), recovery still fails closed on the + // manifest-required flag check. + filesFacade.close(fd); + quarantineDebris(filesFacade, path, "no valid CRC-protected record"); + return null; + } + return new SfManifest(filesFacade, path, fd, selected.generation, + selected.headBase, selected.activeBase); + } catch (Throwable t) { + filesFacade.close(fd); + throw t; + } finally { + Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + } + } + + long activeBase() { + return activeBase; + } + + @Override + public void close() { + if (!closed) { + closed = true; + filesFacade.close(fd); + } + } + + long headBase() { + return headBase; + } + + /** + * Unlinks {@code dir}'s manifest file. Used when a slot is being reset to + * the "nothing durable" state (fresh-start cleanup, close-time drain, or + * recovery accepting a segment-less slot as empty). Returns {@code true} + * when the file is confirmed gone (removed, or never existed). + */ + static boolean removeFile(FilesFacade filesFacade, String dir) { + String path = dir + "/" + FILE_NAME; + return filesFacade.remove(path) || !filesFacade.exists(path); + } + + synchronized void update(long newHeadBase, long newActiveBase) { + if (closed) { + throw new IllegalStateException("SF manifest is closed"); + } + // Committed boundaries only ever move forward: head advances on trim, + // active advances on rotation. Clamp instead of throwing because the + // two writers (producer rotation, manager trim) are serialized on the + // ring monitor but may each compute their argument from a snapshot + // the other has already moved past — e.g. rotation reads the sealed + // list while a trimmed-but-not-yet-removed head segment still sits in + // it. Regressing a durable boundary would let a later crash-recovery + // demand a segment file the trim path already unlinked (startup would + // fail on "missing head segment") or, worse, re-expose stale files + // below an already-committed head. + if (generation > 0) { + if (newHeadBase < headBase) { + newHeadBase = headBase; + } + if (newActiveBase < activeBase) { + newActiveBase = activeBase; + } + } + if (newHeadBase < 0 || newActiveBase < newHeadBase) { + throw new IllegalArgumentException("invalid SF manifest boundaries"); + } + if (generation > 0 && headBase == newHeadBase && activeBase == newActiveBase) { + return; + } + long nextGeneration = generation + 1; + long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buffer, RECORD_SIZE, (byte) 0); + Unsafe.getUnsafe().putInt(buffer, MAGIC); + Unsafe.getUnsafe().putInt(buffer + 4, VERSION); + Unsafe.getUnsafe().putLong(buffer + 8, nextGeneration); + Unsafe.getUnsafe().putLong(buffer + 16, newHeadBase); + Unsafe.getUnsafe().putLong(buffer + 24, newActiveBase); + int crc = Crc32c.update(Crc32c.INIT, buffer, CRC_OFFSET); + Unsafe.getUnsafe().putInt(buffer + CRC_OFFSET, crc); + long offset = (nextGeneration & 1L) * RECORD_SIZE; + if (filesFacade.write(fd, buffer, RECORD_SIZE, offset) != RECORD_SIZE) { + throw new MmapSegmentException("short write updating SF manifest " + path); + } + if (filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync SF manifest " + path); + } + generation = nextGeneration; + headBase = newHeadBase; + activeBase = newActiveBase; + } finally { + Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + } + } + + /** + * Moves creation-crash debris aside so a subsequent exclusive create can + * succeed. Prefers rename (keeps the bytes for postmortem); falls back to + * remove; throws when neither works — leaving the debris in place would + * wedge every subsequent {@link #create}. + */ + private static void quarantineDebris(FilesFacade filesFacade, String path, String reason) { + LOG.warn("SF manifest {} is creation-crash debris ({}); quarantining and starting " + + "from the segment files", path, reason); + if (filesFacade.rename(path, path + ".corrupt") == 0) { + return; + } + if (filesFacade.remove(path)) { + return; + } + throw new MmapSegmentException("could not quarantine invalid SF manifest " + path); + } + + private static Record readRecord(FilesFacade filesFacade, int fd, long buffer, long offset) { + Unsafe.getUnsafe().setMemory(buffer, RECORD_SIZE, (byte) 0); + if (filesFacade.read(fd, buffer, RECORD_SIZE, offset) != RECORD_SIZE) { + return null; + } + if (Unsafe.getUnsafe().getInt(buffer) != MAGIC + || Unsafe.getUnsafe().getInt(buffer + 4) != VERSION) { + return null; + } + int expected = Unsafe.getUnsafe().getInt(buffer + CRC_OFFSET); + int actual = Crc32c.update(Crc32c.INIT, buffer, CRC_OFFSET); + if (expected != actual) { + return null; + } + long generation = Unsafe.getUnsafe().getLong(buffer + 8); + long headBase = Unsafe.getUnsafe().getLong(buffer + 16); + long activeBase = Unsafe.getUnsafe().getLong(buffer + 24); + if (generation <= 0 || headBase < 0 || activeBase < headBase) { + return null; + } + return new Record(generation, headBase, activeBase); + } + + private static final class Record { + private final long activeBase; + private final long generation; + private final long headBase; + + private Record(long generation, long headBase, long activeBase) { + this.generation = generation; + this.headBase = headBase; + this.activeBase = activeBase; + } + } +} diff --git a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java index 248ad61e..194d0972 100644 --- a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java @@ -111,11 +111,21 @@ public int mkdir(String path, int mode) { return Files.mkdir(path, mode); } + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + return Files.mmap(fd, len, offset, flags, memoryTag); + } + @Override public int msync(long addr, long len, boolean async) { return Files.msync(addr, len, async); } + @Override + public void munmap(long address, long len, int memoryTag) { + Files.munmap(address, len, memoryTag); + } + @Override public int openCleanRW(String path) { return Files.openCleanRW(path); @@ -136,6 +146,16 @@ public int openRW(long pathPtr) { return Files.openRW(pathPtr); } + @Override + public int openRWExclusive(String path) { + return Files.openRWExclusive(path); + } + + @Override + public int openRWExclusive(long pathPtr) { + return Files.openRWExclusive(pathPtr); + } + @Override public long length(long pathPtr) { return Files.length(pathPtr); diff --git a/core/src/main/java/io/questdb/client/std/Files.java b/core/src/main/java/io/questdb/client/std/Files.java index 7a4038ed..b0fdfb2e 100644 --- a/core/src/main/java/io/questdb/client/std/Files.java +++ b/core/src/main/java/io/questdb/client/std/Files.java @@ -154,6 +154,24 @@ public static int openRW(long pathPtr) { return openRW0(pathPtr); } + /** + * Atomically creates {@code path} for read-write access. Fails with -1 + * when the path already exists; existing bytes are never truncated. + */ + public static int openRWExclusive(String path) { + long ptr = pathPtr(path); + try { + return openRWExclusive0(ptr); + } finally { + freePathPtr(ptr); + } + } + + /** Native-path variant of {@link #openRWExclusive(String)}. */ + public static int openRWExclusive(long pathPtr) { + return openRWExclusive0(pathPtr); + } + /** * Opens {@code path} for append-only writes, creating it (mode 0644) if * absent. Every {@link #append(int, long, long)} writes at end-of-file @@ -575,6 +593,8 @@ public static void munmap(long address, long len, int memoryTag) { static native int openRW0(long lpszName); + static native int openRWExclusive0(long lpszName); + static native int openAppend0(long lpszName); static native int openCleanRW0(long lpszName); diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java index 60000a3b..082e680e 100644 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java @@ -111,10 +111,18 @@ default int fsyncDir(String dir) { int mkdir(String path, int mode); + default long mmap(int fd, long len, long offset, int flags, int memoryTag) { + return Files.mmap(fd, len, offset, flags, memoryTag); + } + default int msync(long addr, long len, boolean async) { return Files.msync(addr, len, async); } + default void munmap(long address, long len, int memoryTag) { + Files.munmap(address, len, memoryTag); + } + int openCleanRW(String path); /** @@ -130,6 +138,14 @@ default int msync(long addr, long len, boolean async) { /** Variant of {@link #openRW(String)} taking a pre-encoded native UTF-8 path pointer. */ int openRW(long pathPtr); + default int openRWExclusive(String path) { + return Files.openRWExclusive(path); + } + + default int openRWExclusive(long pathPtr) { + return Files.openRWExclusive(pathPtr); + } + /** * Variant of {@code length(String)} taking a pre-encoded native UTF-8 path * pointer; same allocation-elision rationale as {@link #openRW(long)}. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java index 177ee5f6..cff8249b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java @@ -128,7 +128,7 @@ public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception { assertTrue(expected.getMessage(), expected.getMessage().contains("pre-allocation failed")); } - assertEquals("openCleanRW must run exactly once", 1, ff.openCleanRWCalls); + assertEquals("exclusive create must run exactly once", 1, ff.openRWExclusiveCalls); assertEquals("allocate must run exactly once", 1, ff.allocateCalls); assertEquals("fd must be closed on allocate failure", 1, ff.closeCalls); assertEquals("file must be removed on allocate failure", 1, ff.removeCalls); @@ -138,28 +138,31 @@ public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception { } @Test - public void testCreateFailsCleanlyWhenOpenCleanRWReturnsMinusOne() throws Exception { + public void testCreateFailsCleanlyWhenExclusiveOpenReturnsMinusOne() throws Exception { TestUtils.assertMemoryLeak(() -> { String path = tmpDir + "/seg-noopen.sfa"; long sizeBytes = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 64; FaultyFilesFacade ff = new FaultyFilesFacade(); - ff.failOnOpenCleanRW = true; + ff.failOnOpenRWExclusive = true; try { MmapSegment.create(ff, path, 0L, sizeBytes).close(); - fail("expected MmapSegmentException from openCleanRW returning -1"); + fail("expected MmapSegmentException from openRWExclusive returning -1"); } catch (MmapSegmentException expected) { assertTrue(expected.getMessage(), - expected.getMessage().contains("openCleanRW failed")); + expected.getMessage().contains("exclusive create failed")); } - assertEquals("openCleanRW must run exactly once", 1, ff.openCleanRWCalls); - assertEquals("allocate must not run after openCleanRW failure", + assertEquals("exclusive create must run exactly once", 1, ff.openRWExclusiveCalls); + assertEquals("allocate must not run after exclusive-create failure", 0, ff.allocateCalls); assertEquals("close must not be called when no fd was opened", 0, ff.closeCalls); - assertEquals("remove must not be called when openCleanRW failed", + // With O_EXCL semantics a create failure can mean "path already + // exists and belongs to another lifecycle" -- create() must NOT + // unlink a file it never owned. + assertEquals("remove must not be called when exclusive create failed", 0, ff.removeCalls); - assertFalse("no file should exist when openCleanRW failed", + assertFalse("no file should exist when exclusive create failed", Files.exists(path)); }); } @@ -198,7 +201,7 @@ public void testCreateRepeatedAllocateFailuresDoNotAccumulateOrphans() throws Ex } assertEquals("no orphan files may survive repeated allocate failures", 0, survivors); - assertEquals(attempts, ff.openCleanRWCalls); + assertEquals(attempts, ff.openRWExclusiveCalls); assertEquals(attempts, ff.allocateCalls); assertEquals(attempts, ff.closeCalls); assertEquals(attempts, ff.removeCalls); @@ -495,9 +498,29 @@ private static final class FaultyFilesFacade implements FilesFacade { int closeCalls; boolean failOnAllocate; boolean failOnOpenCleanRW; + boolean failOnOpenRWExclusive; int openCleanRWCalls; + int openRWExclusiveCalls; int removeCalls; + @Override + public int openRWExclusive(String path) { + openRWExclusiveCalls++; + if (failOnOpenRWExclusive) { + return -1; + } + return INSTANCE.openRWExclusive(path); + } + + @Override + public int openRWExclusive(long pathPtr) { + openRWExclusiveCalls++; + if (failOnOpenRWExclusive) { + return -1; + } + return INSTANCE.openRWExclusive(pathPtr); + } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java new file mode 100644 index 00000000..21116e68 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java @@ -0,0 +1,827 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Crc32c; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +/** + * Regression tests for the SF recovery fail-open findings: directory + * enumeration errors, per-file open/mmap errors and boundary (leading / + * trailing) segment loss must fail startup without mutating the slot, while + * positively-identified corruption is quarantined only after the surviving + * chain validates. Also pins the crash-window states of the manifest + * protocol (fresh start, rotation, clean-drain) as recoverable. + */ +public class SegmentRecoveryIntegrityTest { + private static final String MANIFEST_NAME = "sf-manifest.bin"; + private static final long SEGMENT_SIZE = 64 * 1024; + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-sf-integrity-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + // ------------------------------------------------------------------ + // Operational failures must fail closed without touching a byte. + // ------------------------------------------------------------------ + + @Test + public void testFindFirstFailureFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when the SF directory cannot be enumerated"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "could not enumerate"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testPartialFindNextFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + private int calls; + + @Override + public int findNext(long findPtr) { + // First advance succeeds, then the listing "fails" the way + // a readdir I/O error does. Recovery must treat the + // partial listing as fatal, not as end-of-directory. + return ++calls >= 2 ? -1 : super.findNext(findPtr); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when the directory listing is incomplete"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "could not fully enumerate"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testOpenRWFailureOnValidSegmentFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String victim = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(victim, 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public int openRW(String path) { + // EMFILE/EACCES-style failure on a perfectly valid file. + return victim.equals(path) ? -1 : super.openRW(path); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when a valid segment cannot be opened"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testMmapFailureOnValidSegmentFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String victim = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(victim, 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + private int victimFd = Integer.MIN_VALUE; + + @Override + public int openRW(String path) { + int fd = super.openRW(path); + if (victim.equals(path)) { + victimFd = fd; + } + return fd; + } + + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + if (fd == victimFd) { + return Files.FAILED_MMAP_ADDRESS; + } + return super.mmap(fd, len, offset, flags, memoryTag); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when a valid segment cannot be mapped"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testUnsupportedVersionFailsRecoveryWithoutQuarantine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String foreign = tmpDir + "/sf-0000000000000001.sfa"; + writeRawSegmentHeader(foreign, MmapSegment.FILE_MAGIC, (byte) 99, 2L); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("a segment written by a different client version must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + // NOT renamed .corrupt: the file belongs to the client build that + // can read it; recovery keeps the slot intact for that writer. + assertDirUnchanged(before); + }); + } + + // ------------------------------------------------------------------ + // Corruption is quarantined, but never before validation decides. + // ------------------------------------------------------------------ + + @Test + public void testCorruptStrayFileQuarantinedInLegacyDirAndSiblingsRecover() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + String stray = tmpDir + "/zz-stray.sfa"; + writeRawSegmentHeader(stray, 0xDEADBEEF, (byte) 1, 0L); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("valid chain must recover around a corrupt stray file", ring); + try { + Assert.assertEquals("all five durable frames must be recovered", + 4, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertFalse("corrupt stray must be quarantined away from the .sfa scan", + Files.exists(stray)); + Assert.assertTrue("quarantine preserves the bytes as evidence", + Files.exists(stray + ".corrupt")); + Assert.assertTrue("legacy recovery must migrate the slot to the manifest", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCorruptChainSegmentWithManifestFailsWithoutMutation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String seg = tmpDir + "/sf-initial.sfa"; + writeSegmentWithFrames(seg, 0, 3); + // Migrate once so the manifest exists and the segment is flagged. + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME)); + // Bit-rot the magic in place. + overwriteInt(seg, 0, 0xBADC0DE); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("losing the only chain segment to corruption must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "corrupt"); + } + // Deferred quarantine: a FAILED recovery must not have renamed, + // deleted, or otherwise mutated anything. + assertDirUnchanged(before); + }); + } + + @Test + public void testFlaggedSegmentWithDeletedManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.assertTrue(Files.remove(tmpDir + "/" + MANIFEST_NAME)); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a manifest-required segment without a manifest must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing"); + } + }); + } + + // ------------------------------------------------------------------ + // Boundary evasion: missing leading/trailing segments must be caught. + // ------------------------------------------------------------------ + + @Test + public void testMissingActiveSegmentWithManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Chain [0..2) exists but the manifest says the active starts at 2. + // Pre-manifest recovery would silently promote the highest present + // segment and hand out overlapping FSNs. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeManifest(1, 0, 2); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a missing trailing/active segment must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF active"); + } + }); + } + + @Test + public void testMissingHeadSegmentWithManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The manifest promises a head at base 0, but only [2..5) survived. + // Pre-manifest recovery would pass contiguity on the remainder and + // silently lose the leading rows. + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + writeManifest(1, 0, 2); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a missing leading/head segment must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF head"); + } + }); + } + + @Test + public void testInteriorGapStillFailsRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000002.sfa", 5, 2); + writeManifest(1, 0, 5); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("an interior FSN gap must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "FSN gap"); + } + }); + } + + // ------------------------------------------------------------------ + // Legal crash-window states must recover without operator action. + // ------------------------------------------------------------------ + + @Test + public void testFreshStartCrashWithTwoEmptyBaseZeroSegmentsRecovers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Fresh engine start provisions sf-initial plus a hot spare, both + // empty at baseSeq 0. A process kill in that window must not brick + // the slot on "ambiguous" empties. + MmapSegment a = MmapSegment.create(tmpDir + "/sf-initial.sfa", 0, SEGMENT_SIZE); + a.close(); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + MmapSegment b = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 0, SEGMENT_SIZE); + b.close(); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("two equivalent empty segments must not brick recovery", ring); + try { + Assert.assertEquals(-1, ring.publishedFsn()); + Assert.assertEquals(0, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + Assert.assertEquals("the redundant empty must have been cleaned up", + 1, countSfaFiles()); + }); + } + + @Test + public void testRotationCrashWindowEmptyActiveAtChainEndRecovers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Crash after rotation committed (manifest fsync'd, spare header + // synced) but before any frame reached the new active: sealed + // chain [0..2) plus an empty active at base 2. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment spare = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + spare.close(); + writeManifest(1, 0, 2); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("freshly-rotated empty active is a legal crash state", ring); + try { + Assert.assertEquals(1, ring.publishedFsn()); + Assert.assertEquals(2, ring.getActive().baseSeq()); + Assert.assertNotNull(ring.firstSealed()); + Assert.assertEquals(0, ring.firstSealed().baseSeq()); + } finally { + ring.close(); + } + }); + } + + @Test + public void testDrainWindowManifestWithoutSegmentsRecoversEmptyAndRemovesManifest() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Clean-drain close unlinks the last segment before the manifest; + // a crash between the two leaves this state. + writeManifest(3, 7, 9); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNull("segment-less slot must recover as EMPTY", ring); + Assert.assertFalse("the stale manifest must be discarded", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testFreshStartCrashBeforeManifestCreationRecoversViaLegacyPath() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Engine crash between creating sf-initial.sfa (unflagged) and + // creating the manifest: recovers via legacy migration. + MmapSegment initial = MmapSegment.create(tmpDir + "/sf-initial.sfa", 0, SEGMENT_SIZE); + initial.close(); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull(ring); + try { + Assert.assertEquals(-1, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue("legacy migration must create the manifest", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + // ------------------------------------------------------------------ + // Engine level: enumeration failure must not truncate the durable log. + // ------------------------------------------------------------------ + + @Test + public void testFindFirstFailureCannotCreateInitialSegment() throws Exception { + TestUtils.assertMemoryLeak(() -> { + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + SegmentManager manager = new SegmentManager( + SEGMENT_SIZE, SegmentManager.DEFAULT_POLL_NANOS, + SegmentManager.UNLIMITED_TOTAL_BYTES, facade); + manager.start(); + try { + try { + new CursorSendEngine(tmpDir, SEGMENT_SIZE, manager).close(); + Assert.fail("startup must fail when SF directory enumeration fails"); + } catch (RuntimeException expected) { + Assert.assertFalse("startup failure created sf-initial.sfa", + Files.exists(tmpDir + "/sf-initial.sfa")); + } + } finally { + manager.close(); + } + }); + } + + @Test + public void testFindFirstFailureDoesNotTruncateExistingLog() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The original failure mode: enumeration error -> treated as empty + // -> fresh start openCleanRW(O_TRUNC) destroys the durable log. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 4); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + SegmentManager manager = new SegmentManager( + SEGMENT_SIZE, SegmentManager.DEFAULT_POLL_NANOS, + SegmentManager.UNLIMITED_TOTAL_BYTES, facade); + manager.start(); + try { + try { + new CursorSendEngine(tmpDir, SEGMENT_SIZE, manager).close(); + Assert.fail("startup must fail when SF directory enumeration fails"); + } catch (RuntimeException expected) { + // expected + } + } finally { + manager.close(); + } + assertDirUnchanged(before); + }); + } + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + /** Creates a real segment at {@code path} and appends {@code frames} 64-byte frames. */ + private static void writeSegmentWithFrames(String path, long baseSeq, int frames) { + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment segment = MmapSegment.create(path, baseSeq, SEGMENT_SIZE); + try { + for (int i = 0; i < frames; i++) { + for (int b = 0; b < 64; b++) { + Unsafe.getUnsafe().putByte(buf + b, (byte) (i * 31 + b)); + } + Assert.assertTrue("test frame must fit", segment.tryAppend(buf, 64) >= 0); + } + } finally { + segment.close(); + } + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } + + /** Writes a raw 64-byte pseudo segment header (+zero padding) for corruption tests. */ + private static void writeRawSegmentHeader(String path, int magic, byte version, long baseSeq) { + int fd = Files.openCleanRW(path); + Assert.assertTrue("could not create " + path, fd >= 0); + try { + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, 64, (byte) 0); + Unsafe.getUnsafe().putInt(buf, magic); + Unsafe.getUnsafe().putByte(buf + 4, version); + Unsafe.getUnsafe().putLong(buf + 8, baseSeq); + Assert.assertEquals(64, Files.write(fd, buf, 64, 0)); + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** Overwrites a single int at {@code offset} in an existing file. */ + private static void overwriteInt(String path, long offset, int value) { + int fd = Files.openRW(path); + Assert.assertTrue("could not open " + path, fd >= 0); + try { + long buf = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putInt(buf, value); + Assert.assertEquals(4, Files.write(fd, buf, 4, offset)); + } finally { + Unsafe.free(buf, 4, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** + * Writes a valid {@code sf-manifest.bin} with one CRC-protected record, + * mirroring SfManifest's on-disk layout (two alternating 64-byte records + * in a 128-byte file; record slot = generation & 1). + */ + private void writeManifest(long generation, long headBase, long activeBase) { + String path = tmpDir + "/" + MANIFEST_NAME; + // openRW (not openCleanRW): callers may layer a second generation's + // record into the sibling slot of an existing manifest. + int fd = Files.openRW(path); + Assert.assertTrue("could not create manifest", fd >= 0); + try { + if (Files.length(path) < 128) { + Assert.assertTrue(Files.truncate(fd, 128)); + } + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, 64, (byte) 0); + Unsafe.getUnsafe().putInt(buf, 0x314d4653); // SFM1 + Unsafe.getUnsafe().putInt(buf + 4, 1); // version + Unsafe.getUnsafe().putLong(buf + 8, generation); + Unsafe.getUnsafe().putLong(buf + 16, headBase); + Unsafe.getUnsafe().putLong(buf + 24, activeBase); + int crc = Crc32c.update(Crc32c.INIT, buf, 60); + Unsafe.getUnsafe().putInt(buf + 60, crc); + long offset = (generation & 1L) * 64; + Assert.assertEquals(64, Files.write(fd, buf, 64, offset)); + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** + * Snapshot of the durable SF payload files (segments, quarantined + * segments, manifest): name -> content. Lifecycle noise such as the slot + * lock and ack watermark is deliberately excluded -- "recovery must not + * mutate the slot" is a statement about the durable log, not about lock + * bookkeeping. + */ + private Map snapshotDir() { + Map out = new HashMap<>(); + Path dir = Paths.get(tmpDir); + try (java.util.stream.Stream stream = java.nio.file.Files.list(dir)) { + stream.filter(java.nio.file.Files::isRegularFile) + .filter(p -> { + String name = p.getFileName().toString(); + return name.endsWith(".sfa") || name.endsWith(".corrupt") + || MANIFEST_NAME.equals(name); + }) + .forEach(p -> { + try { + out.put(p.getFileName().toString(), java.nio.file.Files.readAllBytes(p)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + return out; + } + + /** Asserts the directory holds exactly the snapshotted files, byte for byte. */ + private void assertDirUnchanged(Map before) { + Map after = snapshotDir(); + Assert.assertEquals("file set must be unchanged", before.keySet(), after.keySet()); + for (Map.Entry e : before.entrySet()) { + Assert.assertArrayEquals("bytes of " + e.getKey() + " must be unchanged", + e.getValue(), after.get(e.getKey())); + } + } + + private int countSfaFiles() { + int count = 0; + for (String name : snapshotDir().keySet()) { + if (name.endsWith(".sfa")) { + count++; + } + } + return count; + } + + // ------------------------------------------------------------------ + // Manifest crash windows and record selection + // ------------------------------------------------------------------ + + @Test + public void testCreationCrashZeroByteManifestSelfHeals() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // kill -9 between the manifest's O_EXCL create and its first + // durable record leaves a zero-byte file. No boundary was ever + // committed, so nothing can depend on it: startup must self-heal, + // not demand an operator delete the file. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); + Assert.assertTrue(fd >= 0); + Files.close(fd); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("creation-crash manifest debris must not brick startup", ring); + try { + Assert.assertEquals(2, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue("debris must be quarantined for postmortem", + Files.exists(tmpDir + "/" + MANIFEST_NAME + ".corrupt")); + Assert.assertTrue("a fresh valid manifest must replace the debris", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCreationCrashRecordlessManifestSelfHeals() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Same window, later stage: allocate() completed (128 zero bytes) + // but the first record write/fsync never landed. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); + Assert.assertTrue(fd >= 0); + Assert.assertTrue(Files.truncate(fd, 128)); + Files.close(fd); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("record-less manifest debris must not brick startup", ring); + try { + Assert.assertEquals(2, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME + ".corrupt")); + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testRecordlessManifestWithFlaggedSegmentsFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Flagged segments prove a manifest create COMPLETED at some + // point, so a record-less manifest here is double-slot bit rot, + // not creation debris -- boundaries were lost. Fail closed. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); // truncates + Assert.assertTrue(fd >= 0); + Assert.assertTrue(Files.truncate(fd, 128)); + Files.close(fd); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("boundary loss next to flagged segments must fail closed"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing"); + } + }); + } + + @Test + public void testDrainCrashSurvivingSpareRecoversEmpty() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Clean-drain crash window: boundaries collapsed to head==active, + // the active-base file already unlinked, but an empty hot spare + // (provisional base > activeBase) survived. Everything was acked; + // this must recover as EMPTY, not brick on "missing active". + MmapSegment spare = MmapSegment.create(tmpDir + "/sf-0000000000000007.sfa", 5, SEGMENT_SIZE); + spare.close(); + writeManifest(4, 3, 3); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNull("drain crash window must recover as EMPTY", ring); + Assert.assertEquals("surviving spare must be cleaned up", 0, countSfaFiles()); + Assert.assertFalse("collapsed manifest must be removed", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCorruptActiveWithSameBaseEmptyStandInFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The active (holding unacked frames) is corrupted while a + // leftover spare coincidentally carries the same base. Accepting + // the clean empty as the "rotation crash" active would quarantine + // the unacked frames and re-issue their FSNs -- recovery must + // fail closed instead. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String corruptActive = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(corruptActive, 2, 2); + overwriteInt(corruptActive, 0, 0xBADC0DE); // bit-rot the magic + MmapSegment standIn = MmapSegment.create(tmpDir + "/sf-0000000000000002.sfa", 2, SEGMENT_SIZE); + standIn.close(); + writeManifest(1, 0, 2); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("an empty stand-in must not mask a corrupt active"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF active"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testManifestHigherGenerationRecordWins() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment next = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + next.close(); + // gen1 (slot 1) says active=0; gen2 (slot 0) says active=2. If + // selection picked gen1, the empty at base 2 would sit beyond the + // committed active boundary; gen2 accepts it as the active. + writeManifest(1, 0, 0); + writeManifest(2, 0, 2); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull(ring); + try { + Assert.assertEquals("the higher-generation record must win", + 2, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + }); + } + + @Test + public void testManifestTornNewerRecordFallsBackToOlder() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment next = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + next.close(); + // gen1 (slot 1) is valid and matches the segments; gen2 (slot 0) + // was torn mid-write (bad CRC). Selection must fall back to gen1 + // rather than reject the manifest or trust torn boundaries. + writeManifest(1, 0, 2); + writeManifest(2, 0, 4); + overwriteInt(tmpDir + "/" + MANIFEST_NAME, 60, 0xBADC0DE); // tear gen2's CRC (slot 0) + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("a torn newer record must fall back to the older slot", ring); + try { + Assert.assertEquals(2, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + }); + } + + /** FilesFacade delegating everything to the production instance. */ + private static class DelegatingFacade implements FilesFacade { + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { return INSTANCE.close(fd); } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { return INSTANCE.fsync(fd); } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public int openCleanRW(String path) { return INSTANCE.openCleanRW(path); } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { return INSTANCE.openRW(path); } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { return INSTANCE.remove(path); } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java index 9f533fc7..a226eb0f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java @@ -76,11 +76,17 @@ public void tearDown() { } @Test - public void testRecoveryUnlinksEmptyOrphanSegments() throws Exception { + public void testRecoveryReusesSoleEmptyOrphanAsInitialActive() throws Exception { TestUtils.assertMemoryLeak(() -> { // Simulate a crashed prior session that left an unrotated hot spare // (valid SF01 header, frameCount=0). MmapSegment.create stamps the // header but writes no frames. + // + // Contract (manifest-era): a clean empty leftover is REUSED as the + // fresh ring's initial active instead of being unlinked and + // re-created. The anti-leak guarantee this test was written for + // still holds -- crash cycles converge to exactly one segment + // file, they don't accumulate. String orphanPath = tmpDir + "/sf-orphan.sfa"; MmapSegment empty = MmapSegment.create(orphanPath, 0L, SEGMENT_SIZE); empty.close(); @@ -89,12 +95,17 @@ public void testRecoveryUnlinksEmptyOrphanSegments() throws Exception { SegmentRing recovered = SegmentRing.openExisting(tmpDir, SEGMENT_SIZE); - Assert.assertNull( - "recovery returned a ring even though the only segment was empty", + Assert.assertNotNull( + "recovery must reuse the clean empty leftover as the initial active", recovered); - Assert.assertFalse( - "recovery left the empty orphan .sfa on disk — disk leak grows " - + "with every crash cycle", + try { + Assert.assertEquals("reused ring must be empty", -1, recovered.publishedFsn()); + Assert.assertEquals(0, recovered.getActive().baseSeq()); + Assert.assertEquals(orphanPath, recovered.getActive().path()); + } finally { + recovered.close(); + } + Assert.assertTrue("the reused segment file must still exist", Files.exists(orphanPath)); }); } From c448e34909bec79dd8fb0d666b3133f98319524b Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 14 Jul 2026 00:58:27 +0100 Subject: [PATCH 22/26] fix(client): make SF recovery segment sort O(N log N) unconditionally sortByBaseSeq was a median-of-three Lomuto quicksort. Median-of-three covers the readdir orders a healthy slot produces (lexicographic enumeration yields already-sorted baseSeqs, hashed directory order is effectively random), but Lomuto partitioning is O(N^2) on organ-pipe, duplicate-heavy and median-of-three-killer orders. Exact simulation at the documented 16K-segment ceiling: 22.6M comparisons for organ-pipe, 50.7M for Musser's med3-killer permutation, 134M (full N^2/2) for mass-duplicate baseSeqs -- versus ~221K on the healthy paths. Such orders are only reachable through corrupted-yet-parseable or operator-copied headers, and recovery validation rejects those slots -- but the quadratic stall lands BEFORE validation gets to reject them. The sort is now an introsort: the median-of-three fast path is unchanged, and each root-to-leaf partition path carries a budget of 2*floor(log2(N)) passes. Loop-on-larger iterations count against the budget, so bad-split chains cannot hide in the loop. Ranges that exhaust it fall back to in-place heapsort (still Long.compareUnsigned, zero allocation), capping every adversarial pattern at ~3.8*N*log2(N) comparisons (organ-pipe 773K, duplicates 507K, med3-killer 861K at N=16384). Recursion depth stays under log2(N). The sortComparisons counter ticks +2 per sift-down level so it remains a strict upper bound in the fallback. New adversarial test drives the sort directly with in-memory segments at N=16384 across organ-pipe, all-duplicate, few-distinct, med3-killer and sign-bit-key patterns, asserting unsigned order plus comparisons < 8*N*log2(N) -- >2x headroom over the worst measured pattern, ~12x below the mildest quadratic blow-up. The existing sorted-input regression test is unchanged and still passes. --- .../qwp/client/sf/cursor/SegmentRing.java | 105 +++++++++++++++--- .../qwp/client/sf/cursor/SegmentRingTest.java | 95 ++++++++++++++++ 2 files changed, 186 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 3ad376f3..8b8e752c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -71,8 +71,9 @@ public final class SegmentRing implements QuietCloseable { // openExisting() recovery on this JVM. Used by SegmentRingTest to // assert the sort stays O(N log N) without relying on wall-clock time // (CI runner variance makes elapsed-millisecond bounds flaky). Cheap - // in production: one volatile-free add per partition pass, dwarfed by - // the mmap I/O the recovery does on every segment. + // in production: one volatile-free add per partition pass (plus two per + // sift level in the rare heapsort fallback), dwarfed by the mmap I/O + // the recovery does on every segment. private static long sortComparisons; // References copied while compacting the logical sealed-segment head. // Test-only operation count for deterministic trim-complexity assertions. @@ -1110,9 +1111,10 @@ public static long getNextSealedComparisons() { * {@link #sortByBaseSeq} since the last {@link #resetSortComparisons()} * (or process start). The count is incremented once per partition pass * for the median-of-three pivot pick plus once per element compared - * against the pivot, so a clean run on N segments adds roughly - * {@code 3 + (hi - lo - 1)} per recursive frame, summing to O(N log N). - * Exposed for {@code SegmentRingTest} to detect O(N²) regressions + * against the pivot ({@code 3 + (hi - lo - 1)} per pass), and by two per + * sift-down level when a range falls back to heapsort, so it strictly + * upper-bounds the true compare count and sums to O(N log N) on every + * input. Exposed for {@code SegmentRingTest} to detect O(N²) regressions * deterministically. */ @TestOnly @@ -1145,15 +1147,46 @@ public static void resetTrimMovedReferences() { } /** - * In-place quicksort over {@code list[lo, hi)} keyed by ascending - * {@code baseSeq}. Median-of-three pivot avoids the pathological O(N²) - * on already-sorted input that lexicographic readdir produces (our - * filenames are zero-padded hex of {@code baseSeq}). Recursion depth is - * bounded by ~2 log₂(N) -- for the documented 16K-segment ceiling, well - * under the JVM default stack. + * Drives the recovery-time baseSeq sort over the whole list. Exposed so + * {@code SegmentRingTest} can feed adversarial orders (organ-pipe, mass + * duplicates, median-of-three killer, unsigned-boundary keys) straight + * into the sort and assert comparison bounds without staging thousands + * of segment files on disk. + */ + @TestOnly + public static void sortByBaseSeqForTest(ObjList list) { + sortByBaseSeq(list, 0, list.size()); + } + + /** + * In-place introsort over {@code list[lo, hi)} keyed by ascending + * unsigned {@code baseSeq}. Median-of-three quicksort handles the readdir + * orders a healthy slot produces (lexicographic enumeration of the + * generation-numbered filenames yields already-sorted baseSeqs; hashed + * directory order is effectively random), and a partition-pass budget of + * 2·⌊log₂(N)⌋ demotes any range that keeps splitting badly to in-place + * heapsort. Without that budget, Lomuto with a median-of-three pivot is + * O(N²) on organ-pipe, duplicate-heavy and median-of-three-killer orders + * -- reachable only through corrupted-yet-parseable or operator-copied + * headers, but at the documented 16K-segment ceiling that is 10⁷..10⁸ + * comparisons of startup stall before recovery validation gets to + * reject the slot, so the fallback makes O(N log N) unconditional. + * Recursion depth stays under log₂(N) (recurse on the smaller side, + * loop on the larger), well within the JVM default stack. */ private static void sortByBaseSeq(ObjList list, int lo, int hi) { + int n = hi - lo; + if (n > 1) { + sortByBaseSeq(list, lo, hi, 2 * (31 - Integer.numberOfLeadingZeros(n))); + } + } + + private static void sortByBaseSeq(ObjList list, int lo, int hi, int budget) { while (hi - lo > 1) { + if (budget-- == 0) { + heapSortByBaseSeq(list, lo, hi); + return; + } int mid = (lo + hi) >>> 1; long a = list.get(lo).baseSeq(); long b = list.get(mid).baseSeq(); @@ -1183,17 +1216,61 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) { } swap(list, store, hi - 1); // Recurse on the smaller partition; loop on the larger to keep - // recursion depth bounded by log₂(N). + // recursion depth bounded by log₂(N). Children inherit the + // remaining pass budget: it counts passes along a root-to-leaf + // path, so a chain of bad splits exhausts it after ~2 log₂(N) + // levels no matter how the work is divided. if (store - lo < hi - store - 1) { - sortByBaseSeq(list, lo, store); + sortByBaseSeq(list, lo, store, budget); lo = store + 1; } else { - sortByBaseSeq(list, store + 1, hi); + sortByBaseSeq(list, store + 1, hi, budget); hi = store; } } } + /** + * In-place heapsort over {@code list[lo, hi)} keyed by ascending unsigned + * {@code baseSeq}: the introsort fallback for ranges whose partition-pass + * budget ran out. Guaranteed O(N log N) for any key distribution and any + * initial order; no allocation. + */ + private static void heapSortByBaseSeq(ObjList list, int lo, int hi) { + int n = hi - lo; + for (int root = (n >>> 1) - 1; root >= 0; root--) { + siftDownByBaseSeq(list, lo, root, n); + } + for (int end = n - 1; end > 0; end--) { + swap(list, lo, lo + end); + siftDownByBaseSeq(list, lo, 0, end); + } + } + + private static void siftDownByBaseSeq(ObjList list, int lo, int root, int heapSize) { + while (true) { + int child = (root << 1) + 1; + if (child >= heapSize) { + return; + } + // At most two unsigned compares per level (sibling pick + parent + // test); bump the counter by the constant 2 up front -- same + // cheap-upper-bound convention as the partition pass. + sortComparisons += 2; + if (child + 1 < heapSize + && Long.compareUnsigned(list.get(lo + child).baseSeq(), + list.get(lo + child + 1).baseSeq()) < 0) { + child++; + } + if (Long.compareUnsigned(list.get(lo + root).baseSeq(), + list.get(lo + child).baseSeq()) >= 0) { + return; + } + swap(list, lo + root, lo + child); + root = child; + } + } + private static void swap(ObjList list, int i, int j) { if (i == j) return; MmapSegment tmp = list.get(i); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index 0d31320b..c6466de7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -686,6 +686,101 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { }); } + /** + * Adversarial-order companion to + * {@link #testLargeSegmentCountReopensInOrder}: that test covers the + * already-sorted readdir order median-of-three was chosen for; this one + * covers the orders median-of-three does NOT defend. On the + * pre-introsort quicksort, exact simulation at N=16384 measured ~22.6M + * comparisons for organ-pipe, ~134M (full N²/2) for mass-duplicate + * baseSeqs and ~50.7M for Musser's median-of-three-killer permutation. + * The heapsort fallback caps the worst of these at ~3.8·N·log₂(N) + * (~860K), so the 8·N·log₂(N) bound below keeps >2x headroom against + * harmless implementation drift while sitting ~26x under the mildest + * quadratic blow-up. Also pins unsigned key ordering: baseSeqs with the + * sign bit set must sort above {@code Long.MAX_VALUE}, not below zero. + *

    + * A healthy client cannot produce these orders (recovery validation + * rejects duplicate or non-contiguous baseSeqs moments after the sort), + * but corrupted-yet-parseable headers or operator file copies feed the + * sort BEFORE validation runs, so the sort itself must stay log-linear. + * Sorts in-memory segments directly: staging 16K adversarial header + * files on disk per pattern would dominate the test's runtime without + * adding coverage. + */ + @Test + public void testAdversarialSegmentOrdersSortLogLinear() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int n = 16384; + final long bound = 8L * n * (long) (Math.log(n) / Math.log(2)); + + // Organ pipe: 0,1,...,n/2-1,n/2-1,...,1,0. + long[] organPipe = new long[n]; + for (int i = 0; i < n / 2; i++) { + organPipe[i] = i; + organPipe[n - 1 - i] = i; + } + assertAdversarialSortWithinBound("organ-pipe", organPipe, bound); + + long[] allDuplicates = new long[n]; + for (int i = 0; i < n; i++) { + allDuplicates[i] = 42; + } + assertAdversarialSortWithinBound("all-duplicates", allDuplicates, bound); + + long[] fewDistinct = new long[n]; + for (int i = 0; i < n; i++) { + fewDistinct[i] = i % 4; + } + assertAdversarialSortWithinBound("few-distinct", fewDistinct, bound); + + // Musser's median-of-three killer permutation of 1..n. + long[] med3Killer = new long[n]; + int half = n / 2; + for (int i = 1; i <= half; i++) { + if (i % 2 == 1) { + med3Killer[i - 1] = i; + med3Killer[i] = half + i; + } + med3Killer[half + i - 1] = 2L * i; + } + assertAdversarialSortWithinBound("median-of-three-killer", med3Killer, bound); + + // High-bit keys interleaved with small ones: exercises the + // unsigned comparison contract alongside the comparison bound. + long[] unsignedMix = new long[n]; + for (int i = 0; i < n; i++) { + unsignedMix[i] = (i % 2 == 0) ? (0x8000000000000000L | i) : i; + } + assertAdversarialSortWithinBound("unsigned-mix", unsignedMix, bound); + }); + } + + private static void assertAdversarialSortWithinBound(String label, long[] baseSeqs, long bound) { + final long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1; + ObjList list = new ObjList<>(); + try { + for (long baseSeq : baseSeqs) { + list.add(MmapSegment.createInMemory(baseSeq, segSize)); + } + SegmentRing.resetSortComparisons(); + SegmentRing.sortByBaseSeqForTest(list); + long comparisons = SegmentRing.getSortComparisons(); + for (int i = 1, size = list.size(); i < size; i++) { + assertTrue(label + ": unsigned baseSeq order violated at index " + i, + Long.compareUnsigned(list.get(i - 1).baseSeq(), list.get(i).baseSeq()) <= 0); + } + assertTrue(label + " sort took " + comparisons + " comparisons (expected < " + bound + + " = 8 * N * log2(N) for N=" + baseSeqs.length + + "); regression suggests the introsort heapsort fallback stopped engaging", + comparisons < bound); + } finally { + for (int i = 0, size = list.size(); i < size; i++) { + list.get(i).close(); + } + } + } + @Test public void testRemovingAcknowledgedPrefixMovesLinearReferences() throws Exception { TestUtils.assertMemoryLeak(() -> { From 4b6ba13b06f015bf8b8379be1ad9cab453f8a7b8 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Tue, 14 Jul 2026 09:32:18 +0100 Subject: [PATCH 23/26] Fix client durability and creation shutdown --- .../cutlass/qwp/client/QwpQueryClient.java | 19 ++ .../qwp/client/sf/cursor/AckWatermark.java | 11 +- .../client/sf/cursor/CursorSendEngine.java | 133 ++++---- .../questdb/client/impl/QueryClientPool.java | 54 +++- .../io/questdb/client/impl/SenderPool.java | 68 ++-- .../CursorSendEngineCrashConsistencyTest.java | 180 ++++++++++- .../impl/QueryClientPoolErrorSafetyTest.java | 74 ++++- .../impl/QuestDBImplCloseLifecycleTest.java | 295 ++++++++++++++++++ .../impl/SenderPoolCloseLifecycleTest.java | 17 +- 9 files changed, 713 insertions(+), 138 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index b63b4fa9..f6587631 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -165,6 +165,10 @@ public class QwpQueryClient implements QuietCloseable { private final Random failoverRandom = new Random(); private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; private String authorizationHeader; + // Deterministic lifecycle barrier used by facade shutdown tests. Null in + // production; close() clears and invokes it after winning close ownership + // without allowing hook failures to prevent resource teardown. + private volatile Runnable beforeCloseHook; // Upper bound (ms) on each TCP connect attempt. 0 (default) falls back to // the OS connect timeout. private int connectTimeoutMs = 0; @@ -620,6 +624,16 @@ public void close() { // scratch, double-freeing it. return; } + Runnable hook = beforeCloseHook; + beforeCloseHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable ignored) { + // Omit diagnostics for this test-only hook: even rendering its + // failure must not prevent production resource cleanup. + } + } connected = false; lastCloseTimedOut = false; try { @@ -1004,6 +1018,11 @@ public void seedFailoverRandomForTest(long seed) { } } + @TestOnly + public void setBeforeCloseHookForTest(Runnable hook) { + beforeCloseHook = hook; + } + /** * Returns true if the most recent {@link #close()} call abandoned the I/O thread * because it failed to exit within the join timeout. The native buffer pool and diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index 6e7ebc8c..625fbf22 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -134,9 +134,9 @@ public void close() { /** * Opens (creating if absent) the watermark file in {@code slotDir} * and maps it for the engine's lifetime. Returns {@code null} on - * any setup failure (open fail, mmap fail, unexpected size) — the - * caller falls back to the no-watermark behaviour, no exception - * escapes. Idempotent at the engine layer: a stale file from a + * any setup failure (open fail, mmap fail, unexpected size), leaving + * the caller to choose whether its durability contract permits operation + * without one. Idempotent at the engine layer: a stale file from a * prior session is reused as-is; the first {@link #write(long)} * stamps the magic and the new FSN atomically. */ @@ -167,13 +167,12 @@ static AckWatermark open(FilesFacade filesFacade, String slotDir) { } } if (fd < 0) { - LOG.warn("ack watermark {} could not be opened (rc={}); proceeding without it", - filePath, fd); + LOG.warn("ack watermark {} could not be opened (rc={})", filePath, fd); return null; } long addr = Files.mmap(fd, FILE_SIZE, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { - LOG.warn("ack watermark {} could not be mmapped; proceeding without it", filePath); + LOG.warn("ack watermark {} could not be mmapped", filePath); filesFacade.close(fd); return null; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 9e606b94..90c227fa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -128,11 +128,12 @@ public final class CursorSendEngine implements QuietCloseable { // range with a cumulative self-acknowledge once everything below is // server-acked (CursorWebSocketSendLoop.tryRetireOrphanTail). private long recoveredOrphanTipFsn = -1L; - // Engine-owned mmap'd watermark file. {@code null} in memory mode and - // in disk mode if open() failed (we proceed without it; recovery just - // falls back to lowestBase - 1). Lifetime tied to the engine: opened - // in the constructor, closed by {@link #close()}. The segment manager - // writes through this on every tick where ackedFsn has advanced. + // Engine-owned mmap'd watermark file. {@code null} only in memory mode; + // disk mode fails construction unless the watermark is usable because + // segment-derived recovery cannot distinguish acknowledged residue from + // replayable frames. Lifetime tied to the engine: opened in the + // constructor, closed by {@link #close()}. The segment manager writes + // through this on every tick where ackedFsn has advanced. private final AckWatermark watermark; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized @@ -179,6 +180,11 @@ public final class CursorSendEngine implements QuietCloseable { // With the CAS the worker's cleanup never blocks, so the join returns as // soon as the pass ends. private final AtomicBoolean terminalCleanupClaimed = new AtomicBoolean(); + // True only after a quiescent close reached the final watermark barrier + // and that barrier failed. It permits the shared retry driver to claim + // terminal cleanup; false while cleanup is merely waiting for manager + // quiescence, where an independent retry would race the live worker. + private volatile boolean terminalCleanupRetryReady; // Published only after ring/watermark/unlink cleanup is finished. A close // that loses terminalCleanupClaimed may retry the flock only after this // becomes true, otherwise it could expose the slot while cleanup is live. @@ -337,14 +343,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // - trim ran before persist: segments are gone (so // lowestBase is higher than watermark), watermark // is stale; max picks lowestBase - 1. - // open() returns null on any setup failure so a missing - // mmap doesn't take down the engine -- we just fall - // back to the bare lowestBase - 1 seed. + // Segment-derived state cannot tell whether frames in the + // lowest surviving segment were acknowledged. Starting + // without the watermark would therefore expose acknowledged + // residue for replay, so disk recovery must fail closed when + // the file cannot be opened or mapped. watermarkInProgress = AckWatermark.open(filesFacade, sfDir); + if (watermarkInProgress == null) { + throw new IllegalStateException( + "could not open required ack watermark for SF slot " + sfDir); + } long baseSeed = lowestBase - 1; - long watermarkFsn = watermarkInProgress != null - ? watermarkInProgress.read() - : AckWatermark.INVALID; + long watermarkFsn = watermarkInProgress.read(); // Reject watermarks past publishedFsn: a correctly // operating prior session cannot have produced one, so // a value above the on-disk frame ceiling is corruption @@ -393,6 +403,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(filesFacade, sfDir); watermarkInProgress = AckWatermark.open(filesFacade, sfDir); + if (watermarkInProgress == null) { + throw new IllegalStateException( + "could not open required ack watermark for SF slot " + sfDir); + } } MmapSegment initial; String initialPath = null; @@ -782,35 +796,36 @@ public synchronized void close() { * against a close() that holds the monitor while joining the worker. */ private void finishClose(boolean fullyDrained) { - try { - RuntimeException durabilityFailure = null; - // On a fully-drained close, persist the final acked FSN through - // the still-mapped watermark BEFORE closing the ring/watermark - // and BEFORE unlinking any segment file. The manager persists - // the watermark only on its own tick, so it may lag the final - // ack. If the unlink below then fails (or the process dies - // mid-unlink), residual acknowledged .sfa files without an - // up-to-date watermark would seed the successor's recovery at - // lowestBase - 1 and replay already-acknowledged rows -- - // duplicates on a non-DEDUP table. The write is a single mmap - // store, so it succeeds even when the unlink is about to fail - // (e.g. the slot dir turned read-only). Quiescence is already - // established here, so no manager tick can race this write. - if (fullyDrained && watermark != null) { - try { - long finalAckedFsn = ring.ackedFsn(); - if (finalAckedFsn >= 0) { - watermark.write(finalAckedFsn); - watermark.sync(); - if (filesFacade.fsyncDir(sfDir) != 0) { - throw new IllegalStateException( - "could not fsync SF slot directory before segment cleanup"); - } + // On a fully-drained close, persist the final acked FSN through the + // still-mapped watermark BEFORE closing the ring/watermark and BEFORE + // unlinking any segment file. The manager persists the watermark only + // on its own tick, so it may lag the final ack. If any part of this + // covering barrier fails, retain the ring, watermark and slot flock: + // publishing the slot would let a successor recover acknowledged + // residue from a stale durable watermark. The shared retry driver owns + // convergence after the one-shot public close reports the failure. + if (fullyDrained && watermark != null) { + try { + long finalAckedFsn = ring.ackedFsn(); + if (finalAckedFsn >= 0) { + watermark.write(finalAckedFsn); + watermark.sync(); + if (filesFacade.fsyncDir(sfDir) != 0) { + throw new IllegalStateException( + "could not fsync SF slot directory before segment cleanup"); } - } catch (RuntimeException e) { - durabilityFailure = e; } + } catch (RuntimeException | Error e) { + terminalCleanupRetryReady = true; + terminalCleanupClaimed.set(false); + startFlockReleaseRetry(); + throw e; } + } + terminalCleanupRetryReady = false; + + try { + RuntimeException durabilityFailure = null; try { ring.close(); } catch (Throwable ignored) { @@ -829,7 +844,7 @@ private void finishClose(boolean fullyDrained) { } catch (Throwable ignored) { } } - if (fullyDrained && watermark != null && durabilityFailure == null) { + if (fullyDrained && watermark != null) { boolean segmentsRemoved = false; try { segmentsRemoved = unlinkAllSegmentFiles(filesFacade, sfDir); @@ -852,17 +867,15 @@ private void finishClose(boolean fullyDrained) { + "engine on this slot recovers them as fully acked and retries the " + "unlink on its own close", sfDir); } - } else if (fullyDrained && watermark == null && sfDir != null) { - LOG.warn("close-time segment cleanup skipped on slot {} because no ack watermark " - + "is available to cover a host crash during unlink", sfDir); } if (durabilityFailure != null) { throw durabilityFailure; } } finally { - // Reaching finishClose at all requires established quiescence, so - // releasing the flock is safe even if a step above threw. Leaking - // it would strand the slot until process exit for no reason. + // The final watermark covering barrier has succeeded, so terminal + // resources can be released even if later best-effort cleanup or + // its post-unlink directory sync failed. The durable watermark + // still covers any segment a host crash restores. // // ORDER MATTERS: explicitly release the flock, verify it, and // only then publish closeCompleted. Pools read isCloseCompleted() @@ -973,7 +986,19 @@ private Runnable createDeferredClose() { } private boolean retryFlockReleaseIfReady() { - if (closeCompleted || !terminalResourcesCleaned) { + if (closeCompleted) { + return true; + } + if (!terminalResourcesCleaned) { + if (!terminalCleanupRetryReady + || !terminalCleanupClaimed.compareAndSet(false, true)) { + return false; + } + try { + finishClose(fullyDrainedForDeferredClose); + } catch (Throwable ignored) { + return false; + } return closeCompleted; } boolean released; @@ -1091,8 +1116,8 @@ private void startFlockReleaseRetry() { } } if (startFailure == null) { - LOG.error("SF slot flock release failed during engine close; keeping " - + "closeCompleted=false and retrying on the shared driver so " + LOG.error("SF terminal cleanup or slot flock release failed during engine close; " + + "keeping closeCompleted=false and retrying on the shared driver so " + "retired capacity recovers after the transient failure [slot={}]", sfDir == null ? "" : sfDir); } else { @@ -1141,10 +1166,11 @@ public void setSlotLockReleaseListener(Runnable listener) { } /** - * Re-arms the shared flock-release retry for an engine whose terminal - * cleanup finished but whose confirmed flock release is still pending - * and no longer scheduled — the retry driver thread failed to start when - * the release first failed (e.g. OOM at thread creation). + * Re-arms the shared terminal retry for an engine whose final watermark + * barrier or confirmed flock release is still pending and no longer + * scheduled because the retry driver thread failed to start (e.g. OOM at + * thread creation). A close still waiting for worker quiescence cannot be + * retried here because terminal cleanup may not race that worker. * {@code Sender.close()} is one-shot by contract, so pool probes * ({@code QwpWebSocketSender.isSlotLockReleased()}) call this to keep a * retired slot's capacity recoverable instead of lost until process @@ -1153,10 +1179,9 @@ public void setSlotLockReleaseListener(Runnable listener) { * may call it under their capacity lock. */ public void ensureFlockReleaseRetryScheduled() { - if (closeCompleted || !terminalResourcesCleaned) { - return; + if (!closeCompleted && (terminalResourcesCleaned || terminalCleanupRetryReady)) { + startFlockReleaseRetry(); } - startFlockReleaseRetry(); } /** diff --git a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java index 259ff10c..da9249c5 100644 --- a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java +++ b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java @@ -60,6 +60,10 @@ public final class QueryClientPool implements AutoCloseable { private final ArrayList all; private final ArrayDeque available; private final String configurationString; + // Signals completion of internally owned creation lifecycles. Kept + // separate from workerReleased so an acquire waiter cannot consume the only + // wakeup intended for close(). + private final Condition creationFinished; // Test seam. Production connects via QwpQueryClient.connect(); white-box // tests in io.questdb.client.test.impl reach the package-private constructor // by reflection to inject a hook that throws a non-RuntimeException @@ -146,6 +150,7 @@ public QueryClientPool( this.maxLifetimeMillis = maxLifetimeMillis; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); + this.creationFinished = lock.newCondition(); this.workerReleased = lock.newCondition(); int built = 0; // Tracks a worker built by createUnlocked() but not yet added to `all`: @@ -231,16 +236,11 @@ public QueryWorker acquire() { // incremented, permanently shrinking pool capacity until // every acquire() times out. Restoring the reservation // for any throwable is safe. - lock.lock(); - inFlightCreations--; - workerReleased.signal(); - lock.unlock(); // createUnlocked() returns a fully connected client // (socket + native scratch + I/O thread), so if start() // threw afterwards we must close it here -- nothing else - // references it. createUnlocked() already self-cleans - // when connect() throws, leaving created == null, so - // this only fires on the start()-throws path. + // references it. Keep the creation reservation until + // that cleanup finishes so close() cannot return first. if (created != null) { try { created.shutdown(); @@ -249,33 +249,45 @@ public QueryWorker acquire() { // original creation failure rethrown below. } } + lock.lock(); + try { + inFlightCreations--; + creationFinished.signalAll(); + workerReleased.signal(); + } finally { + lock.unlock(); + } throw new QueryException((byte) 0, "failed to create query client: " + e.getMessage(), e); } lock.lock(); - inFlightCreations--; if (closed) { - // Pool was closed mid-creation -- tear the fresh worker - // down rather than leaking it, but OUTSIDE the lock: - // shutdown() joins the dispatch thread for up to - // SHUTDOWN_JOIN_MILLIS, and close()/release()/discard()/ - // cancelIfCurrent() all contend on this lock (whose - // contract is "held only briefly"). The accounting above - // already ran under the lock, and the worker never - // entered `all`, so close()'s snapshot loop cannot race - // this teardown. + // Keep inFlightCreations reserved while shutdown runs + // outside the lock. The worker never entered `all`, so + // this reservation is close()'s sole ownership record. lock.unlock(); try { created.shutdown(); } catch (Throwable ignored) { // Best-effort: an Error from teardown must not mask // the closed-pool signal. + } finally { + lock.lock(); + try { + inFlightCreations--; + creationFinished.signalAll(); + workerReleased.signalAll(); + } finally { + lock.unlock(); + } } throw new QueryException((byte) 0, "QuestDB handle is closed"); } all.add(created); // Stamp the first lease id for this freshly built worker. created.bumpGeneration(); + inFlightCreations--; + creationFinished.signalAll(); return created; } if (remainingNanos <= 0) { @@ -307,6 +319,14 @@ public void close() { } closed = true; workerReleased.signalAll(); + // A reservation owns every resource created outside the lock until + // the worker is either published in `all` or fully torn down. No + // user ever received these workers, so close() must not apply an + // abandoned-lease timeout. Preserve interrupts while waiting for + // the internal ownership count to reach zero. + while (inFlightCreations > 0) { + creationFinished.awaitUninterruptibly(); + } snapshot = new ArrayList<>(all); } finally { lock.unlock(); diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 1bd60f3f..32540026 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -107,6 +107,10 @@ public final class SenderPool implements AutoCloseable { private final ArrayList all; private final ArrayDeque available; private final String configurationString; + // Signals completion of internally owned creation lifecycles. Kept + // separate from slotReleased so a capacity waiter cannot consume the only + // wakeup intended for close(). + private final Condition creationFinished; // User-supplied ingest callbacks, shared across every pooled Sender this // pool builds. Null -> each sender keeps its loud-not-silent default. private final SenderConnectionListener connectionListener; @@ -369,6 +373,7 @@ private SenderPool( this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); this.retiredSlots = new ArrayList<>(maxSize); + this.creationFinished = lock.newCondition(); this.slotReleased = lock.newCondition(); // Probe the config once, up front: this validates it eagerly (so a // bad config fails at construction even when minSize == 0) and tells @@ -810,33 +815,26 @@ public PooledSender borrow() { // idempotent, so undoing the reservation for any // throwable is safe. lock.lock(); - inFlightCreations--; - freeSlotIndex(slotIndex); - slotReleased.signal(); - lock.unlock(); + try { + inFlightCreations--; + freeSlotIndex(slotIndex); + creationFinished.signalAll(); + slotReleased.signal(); + } finally { + lock.unlock(); + } throw e; } lock.lock(); - inFlightCreations--; if (closed) { - // Pool was closed mid-creation -- destroy the new connection - // rather than leaking it. Other waiters have been signaled - // by close() already. The delegate is closed OUTSIDE the - // lock (mirroring retireLease): its close() can block for - // seconds (bounded ack drain, drainer-pool wind-down) or - // longer (unbounded I/O-thread latch await behind an - // OS-level connect), which held here would stall close(), - // giveBack/retireLease and reapIdle behind the pool lock. - // Accounting first, under the lock: for an SF slot the - // index reservation moves from inFlightCreations to - // closingSlots until the close below releases the flock, - // and pendingLeaseTeardowns keeps the out-of-lock close - // visible to close()'s outstanding-teardown wait. + // Pool was closed mid-creation. Keep inFlightCreations + // reserved while the delegate is closed outside the + // lock: close() waits on that counter, so the creation + // remains internally owned until its teardown completes. boolean reserved = created.slotIndex() >= 0; if (reserved) { closingSlots++; } - pendingLeaseTeardowns++; lock.unlock(); try { created.delegate().close(); @@ -844,23 +842,24 @@ public PooledSender borrow() { // Best-effort: an Error (e.g. -ea AssertionError) // from teardown must not mask the closed-pool signal. } finally { - // Re-lock to reclaim the SF slot index and signal a - // close() waiting on this teardown. MUST run even if - // the delegate close threw, otherwise the slot stays - // reserved forever and close() waits out its full - // budget on a teardown that already happened. lock.lock(); - pendingLeaseTeardowns--; - if (reserved) { - reclaimSlot(created, " after closed-mid-creation teardown"); + try { + if (reserved) { + reclaimSlot(created, " after closed-mid-creation teardown"); + } + inFlightCreations--; + creationFinished.signalAll(); + slotReleased.signalAll(); + } finally { + lock.unlock(); } - slotReleased.signalAll(); - lock.unlock(); } throw new LineSenderException("QuestDB handle is closed"); } all.add(created); created.bumpGeneration(); + inFlightCreations--; + creationFinished.signalAll(); return new PooledSender(created, created.generation()); } // Capacity-starved: re-probe retired slots BEFORE the terminal @@ -938,6 +937,8 @@ void markClosing() { * free their native memory under its feet -- a use-after-free / SEGV, not * an exception (C1). Instead: *

      + *
    1. waits for every internally owned creation to publish or complete its + * closed-mid-creation teardown; then
      2. *
    2. waits boundedly (up to {@code acquireTimeoutMillis}, hard-capped at * {@link #MAX_CLOSE_LEASE_WAIT_MILLIS}) for outstanding leases to come home -- {@link #giveBack} and {@link #discardBroken} * observe {@code closed} and tear each delegate down on the returning @@ -964,6 +965,15 @@ public void close() { closed = true; // Wake parked borrowers so they observe the shutdown and throw. slotReleased.signalAll(); + // A creation reservation represents pool-owned resources from the + // moment borrow() leaves the lock through either publication or + // completed cleanup. Unlike an abandoned user-visible lease, this + // ownership cannot be leaked after close() returns, so wait without + // a timeout. awaitUninterruptibly preserves the interrupt flag while + // maintaining the shutdown ownership invariant. + while (inFlightCreations > 0) { + creationFinished.awaitUninterruptibly(); + } // Bounded graceful wait for outstanding leases. A slot is borrowed // iff it is in `all` but not in `available`; retireLease's // delegate-close section (running outside the lock on a returning diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java index d6c3f665..a7017961 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java @@ -45,9 +45,9 @@ public class CursorSendEngineCrashConsistencyTest { @Test - public void testCloseDurabilityOrderAndSyncFailurePropagation() throws Exception { + public void testCloseDurabilityOrderAndPostCleanupSyncFailurePropagation() throws Exception { TestUtils.assertMemoryLeak(() -> { - for (int failAt : new int[]{-1, 0, 1, 2, 4}) { + for (int failAt : new int[]{-1, 4}) { String root = Paths.get(System.getProperty("java.io.tmpdir"), "qdb-close-crash-" + failAt + "-" + System.nanoTime()).toString(); String slot = root + "/slot"; @@ -69,24 +69,16 @@ public void testCloseDurabilityOrderAndSyncFailurePropagation() throws Exception ff.beginClose(); try { engine.close(); - if (failAt >= 0 && failAt != 3) { + if (failAt >= 0) { Assert.fail("sync failure was swallowed at boundary " + failAt); } } catch (IllegalStateException expected) { - Assert.assertTrue("unexpected close failure: " + expected, - failAt >= 0 && failAt != 3); + Assert.assertTrue("unexpected close failure: " + expected, failAt >= 0); } engine = null; - if (failAt >= 0 && failAt <= 2) { - Assert.assertFalse("segment deletion started after watermark barrier failure", - ff.events.contains("segment-remove")); - Assert.assertTrue("watermark was removed after its durability barrier failed", - Files.exists(slot + "/" + AckWatermark.FILE_NAME)); - } else { - Assert.assertFalse("simulated crash replays acknowledged rows at boundary " + failAt, - ff.durableSegments && !ff.durableWatermark); - } + Assert.assertFalse("simulated crash replays acknowledged rows at boundary " + failAt, + ff.durableSegments && !ff.durableWatermark); if (failAt == -1) { Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync", "dir-fsync", "segment-remove", "dir-fsync", "watermark-remove"), @@ -105,6 +97,138 @@ public void testCloseDurabilityOrderAndSyncFailurePropagation() throws Exception }); } + @Test + public void testFinalWatermarkBarrierFailureRetainsSlotUntilRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (String barrier : Arrays.asList("watermark-msync", "watermark-fsync", "dir-fsync")) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-close-barrier-" + barrier + "-" + System.nanoTime()).toString(); + String slot = root + "/slot"; + SegmentManager manager = null; + CursorSendEngine predecessor = null; + CursorSendEngine successor = null; + CrashImageFilesFacade ff = null; + long payload = 0; + Throwable failure = null; + try { + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + ff = new CrashImageFilesFacade(slot, -1); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60), + SegmentManager.UNLIMITED_TOTAL_BYTES, ff); + payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + predecessor = new CursorSendEngine(slot, segmentSize, manager); + Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7); + Assert.assertEquals(0L, predecessor.appendBlocking(payload, 32)); + Assert.assertTrue(predecessor.acknowledge(0L)); + + ff.failPersistently(barrier); + try { + predecessor.close(); + Assert.fail("expected close to report failed final barrier " + barrier); + } catch (IllegalStateException expected) { + Assert.assertTrue("unexpected close failure: " + expected, + expected.getMessage().contains("ack watermark") + || expected.getMessage().contains("slot directory")); + } + try { + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.fail("successor acquired a slot whose final ACK barrier failed: " + barrier); + } catch (IllegalStateException expected) { + Assert.assertTrue("successor failed for the wrong reason: " + expected, + expected.getMessage().contains("slot already in use")); + } + + ff.clearPersistentFailure(); + awaitCloseCompleted(predecessor); + predecessor = null; + + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertEquals("successful retry must leave no ACKed frame to replay", + -1L, successor.publishedFsn()); + successor.close(); + Assert.assertTrue(successor.isCloseCompleted()); + successor = null; + } catch (Throwable t) { + failure = t; + } finally { + if (failure != null && ff != null) { + // Let a retained predecessor converge before cleanup. + // The exact red assertion remains attached to failure. + ff.clearPersistentFailure(); + } + failure = closeEngine(failure, successor); + failure = closeEngine(failure, predecessor); + failure = closeManager(failure, manager); + failure = freePayload(failure, payload); + failure = removeRoot(failure, root); + } + rethrow(failure); + } + }); + } + + @Test + public void testRecoveredSlotRejectsMissingWatermark() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-missing-watermark-" + System.nanoTime()).toString(); + String slot = root + "/slot"; + SegmentManager manager = null; + CursorSendEngine predecessor = null; + CursorSendEngine successor = null; + long payload = 0; + Throwable failure = null; + try { + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + CrashImageFilesFacade ff = new CrashImageFilesFacade(slot, -1); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60), + SegmentManager.UNLIMITED_TOTAL_BYTES, ff); + payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + predecessor = new CursorSendEngine(slot, segmentSize, manager); + Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7); + Assert.assertEquals(0L, predecessor.appendBlocking(payload, 32)); + Assert.assertTrue(predecessor.acknowledge(0L)); + ff.beginClose(); + ff.blockSegmentRemove = true; + predecessor.close(); + Assert.assertTrue(predecessor.isCloseCompleted()); + predecessor = null; + Assert.assertTrue("precondition: ACKed segment residue must survive", + Files.exists(slot + "/sf-initial.sfa")); + + ff.failWatermarkOpen = true; + try { + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.fail("recovered disk slot started without a usable ACK watermark"); + } catch (IllegalStateException expected) { + Assert.assertTrue("successor failed for the wrong reason: " + expected, + expected.getMessage().contains("ack watermark")); + } + + ff.failWatermarkOpen = false; + ff.blockSegmentRemove = false; + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertTrue(successor.wasRecoveredFromDisk()); + Assert.assertTrue("successor exposes predecessor's ACKed frame", + successor.ackedFsn() >= successor.publishedFsn()); + successor.close(); + Assert.assertTrue(successor.isCloseCompleted()); + successor = null; + } catch (Throwable t) { + failure = t; + } finally { + failure = closeEngine(failure, successor); + failure = closeEngine(failure, predecessor); + failure = closeManager(failure, manager); + failure = freePayload(failure, payload); + failure = removeRoot(failure, root); + } + rethrow(failure); + }); + } + private static Throwable addCleanupFailure(Throwable failure, Throwable cleanupFailure) { if (failure == null) { return cleanupFailure; @@ -115,6 +239,14 @@ private static Throwable addCleanupFailure(Throwable failure, Throwable cleanupF return failure; } + private static void awaitCloseCompleted(CursorSendEngine engine) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!engine.isCloseCompleted() && System.nanoTime() < deadline) { + Thread.yield(); + } + Assert.assertTrue("close retry did not complete", engine.isCloseCompleted()); + } + private static Throwable closeEngine(Throwable failure, CursorSendEngine engine) { if (engine != null) { try { @@ -196,9 +328,12 @@ private static final class CrashImageFilesFacade implements FilesFacade { private final int failAt; private final String slot; private boolean active; + private boolean blockSegmentRemove; private boolean durableSegments = true; private boolean durableWatermark; private int eventIndex; + private boolean failWatermarkOpen; + private String persistentFailure; private int watermarkFd = -1; private CrashImageFilesFacade(String slot, int failAt) { @@ -212,9 +347,20 @@ private void beginClose() { eventIndex = 0; } + private void clearPersistentFailure() { + persistentFailure = null; + } + private boolean fail(String event) { events.add(event); - return eventIndex++ == failAt; + return event.equals(persistentFailure) || eventIndex++ == failAt; + } + + private void failPersistently(String event) { + active = true; + events.clear(); + eventIndex = 0; + persistentFailure = event; } @Override @@ -269,6 +415,7 @@ public int msync(long addr, long len, boolean async) { } @Override public int openCleanRW(String path) { + if (failWatermarkOpen && path.equals(slot + "/" + AckWatermark.FILE_NAME)) return -1; int fd = INSTANCE.openCleanRW(path); if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; return fd; @@ -277,6 +424,7 @@ public int openCleanRW(String path) { public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } @Override public int openRW(String path) { + if (failWatermarkOpen && path.equals(slot + "/" + AckWatermark.FILE_NAME)) return -1; int fd = INSTANCE.openRW(path); if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; return fd; @@ -290,7 +438,7 @@ public long read(int fd, long addr, long len, long offset) { @Override public boolean remove(String path) { if (active && path.endsWith(".sfa")) { - if (fail("segment-remove")) return false; + if (blockSegmentRemove || fail("segment-remove")) return false; } else if (active && path.equals(slot + "/" + AckWatermark.FILE_NAME)) { if (fail("watermark-remove")) return false; } diff --git a/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java index bc36d1ae..ca5e33aa 100644 --- a/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java @@ -34,10 +34,13 @@ import org.junit.Assert; import org.junit.Test; +import java.lang.reflect.Field; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; // Error-safety of the three QueryClientPool creation paths the teardown-hardening @@ -230,16 +233,31 @@ public void preWarmDoesNotLeakNativeScratchOnErrorFromStart() throws Exception { // the dispatch thread for up to SHUTDOWN_JOIN_MILLIS; cancelIfCurrent's // contract is that this lock is "held only briefly") -- and still reclaim // everything: the client's NATIVE_DEFAULT scratch and the creation slot. - // RED (teardown dropped in the restructure): the scratch leaks and the - // baseline assertion fails. - // GREEN: shutdown() runs after the lock is released -> no leak, accounting - // restored, acquire() surfaces the closed pool. + // RED (rendering a throwing before-close hook's failure also throws after + // close ownership is won): QueryWorker suppresses that second failure, + // the pool releases its creation reservation, and the client's native + // scratch remains live. + // GREEN: close() suppresses the test-witness failure without rendering it, + // completes production teardown, and only then lets the pool release + // the reservation. @Test(timeout = 30_000) public void closedMidCreationTearsDownFreshWorkerWithoutLeak() throws Exception { TestUtils.assertMemoryLeak(() -> { CountDownLatch inConnect = new CountDownLatch(1); CountDownLatch releaseConnect = new CountDownLatch(1); + AtomicInteger closeHookCalls = new AtomicInteger(); + AtomicReference createdClient = new AtomicReference<>(); Consumer connectHook = client -> { + createdClient.set(client); + client.setBeforeCloseHookForTest(() -> { + closeHookCalls.incrementAndGet(); + throw new AssertionError("injected query close hook failure") { + @Override + public String toString() { + throw new AssertionError("injected throwable rendering failure"); + } + }; + }); inConnect.countDown(); try { if (!releaseConnect.await(10, TimeUnit.SECONDS)) { @@ -268,25 +286,35 @@ public void closedMidCreationTearsDownFreshWorkerWithoutLeak() throws Exception Assert.assertTrue("acquirer never reached connect()", inConnect.await(10, TimeUnit.SECONDS)); - // Close the pool while the worker build is in flight (the fresh - // worker never entered `all`, so close()'s snapshot skips it), - // then let the build finish: acquire() must observe `closed`, - // tear the worker down on its own thread and throw. - pool.close(); + // Close the pool while the worker build is in flight. close() + // now waits for the internal creation reservation, so run it + // concurrently and wait until it parks on that reservation + // before releasing connect(). + Thread closer = new Thread(pool::close, "mid-creation-pool-closer"); + closer.start(); + awaitCreationWaiter(pool); + + // Let the build finish: acquire() must observe `closed`, tear + // the worker down on its own thread and throw. close() returns + // only after that teardown releases the reservation. releaseConnect.countDown(); acquirer.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); Assert.assertFalse("acquirer did not finish", acquirer.isAlive()); + Assert.assertFalse("pool close did not finish", closer.isAlive()); Assert.assertTrue( "acquire() must surface the closed pool, got: " + acquireOutcome.get(), acquireOutcome.get() instanceof QueryException && String.valueOf(acquireOutcome.get().getMessage()).contains("closed")); + Assert.assertEquals("pool released the creation reservation after teardown", + 0, inFlightCreations(pool)); long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT); Assert.assertEquals( - "closed-mid-creation acquire() leaked the fresh worker's NATIVE_DEFAULT scratch", + "throwing close hook prevented fresh worker teardown before reservation release", baseline, after); - Assert.assertEquals("in-flight creation accounting must be restored", - 0, inFlightCreations(pool)); + createdClient.get().close(); + Assert.assertEquals("close hook must be one-shot", 1, closeHookCalls.get()); } finally { pool.close(); } @@ -299,6 +327,28 @@ private static Consumer alwaysThrow() { }; } + private static void awaitCreationWaiter(QueryClientPool pool) throws Exception { + Field lockField = QueryClientPool.class.getDeclaredField("lock"); + Field conditionField = QueryClientPool.class.getDeclaredField("creationFinished"); + lockField.setAccessible(true); + conditionField.setAccessible(true); + ReentrantLock lock = (ReentrantLock) lockField.get(pool); + Condition creationFinished = (Condition) conditionField.get(pool); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + lock.lock(); + try { + if (lock.hasWaiters(creationFinished)) { + return; + } + } finally { + lock.unlock(); + } + Thread.yield(); + } + Assert.fail("pool close did not wait on the creation reservation"); + } + // connectHook that connects nothing: fromConfig() has already committed the // NATIVE_DEFAULT scratch, so the client is half-built (scratch, no socket) // -- exactly the state createUnlocked() returns before start() runs. diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java new file mode 100644 index 00000000..dd697049 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java @@ -0,0 +1,295 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.impl; + +import io.questdb.client.QueryException; +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.impl.QueryClientPool; +import io.questdb.client.impl.QuestDBImpl; +import io.questdb.client.impl.SenderPool; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.IntFunction; + +public class QuestDBImplCloseLifecycleTest { + + private static final String QUERY_CFG = "ws::addr=127.0.0.1:9000;"; + private static final String SENDER_CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + + @Test(timeout = 30_000) + public void facadeCloseWaitsForQueryCreationAndTeardown() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + CountDownLatch inTeardown = new CountDownLatch(1); + CountDownLatch releaseTeardown = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + Consumer connectHook = client -> { + client.setBeforeCloseHookForTest(() -> { + inTeardown.countDown(); + awaitOrFail(releaseTeardown, "test never released query teardown"); + teardownCount.incrementAndGet(); + }); + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released query creation"); + }; + QuestDBImpl db = newQuestDB(0, 0, slotIndex -> fakeSender(null, null, null), connectHook); + AtomicReference borrowOutcome = new AtomicReference<>(); + AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); + Thread borrower = new Thread(() -> { + try { + db.borrowQuery(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "facade-query-borrower"); + Thread closer = new Thread(() -> { + db.close(); + closeReturnedInterrupted.set(Thread.currentThread().isInterrupted()); + }, "facade-query-closer"); + try { + borrower.start(); + Assert.assertTrue("query borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + + closer.start(); + QueryClientPool pool = (QueryClientPool) getField(db, "queryPool"); + awaitBooleanField(pool, "closed"); + awaitCreationWaiter(pool, + "facade close did not wait while query construction was internally owned"); + closer.interrupt(); + awaitCreationWaiter(pool, + "interrupt allowed facade close to abandon query creation ownership"); + + releaseCreation.countDown(); + Assert.assertTrue("query borrow never reached closed-mid-creation teardown", + inTeardown.await(10, TimeUnit.SECONDS)); + awaitCreationWaiter(pool, + "facade close abandoned query ownership during internal teardown"); + + releaseTeardown.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("query borrower did not finish", borrower.isAlive()); + Assert.assertFalse("facade close did not finish", closer.isAlive()); + Assert.assertEquals("internally owned query client must be torn down exactly once", + 1, teardownCount.get()); + Assert.assertTrue("facade close must preserve interruption after internal teardown", + closeReturnedInterrupted.get()); + Assert.assertTrue("borrowQuery() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof QueryException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + releaseTeardown.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseWaitsForSenderCreationAndTeardown() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + CountDownLatch inTeardown = new CountDownLatch(1); + CountDownLatch releaseTeardown = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + IntFunction senderFactory = slotIndex -> { + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released sender creation"); + return fakeSender(teardownCount, inTeardown, releaseTeardown); + }; + QuestDBImpl db = newQuestDB(0, 0, senderFactory, client -> { + }); + AtomicReference borrowOutcome = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + db.borrowSender(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "facade-sender-borrower"); + Thread closer = new Thread(db::close, "facade-sender-closer"); + try { + borrower.start(); + Assert.assertTrue("sender borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + + closer.start(); + SenderPool pool = (SenderPool) getField(db, "senderPool"); + awaitBooleanField(pool, "closeStarted"); + awaitCreationWaiter(pool, + "facade close did not wait while sender construction was internally owned"); + + releaseCreation.countDown(); + Assert.assertTrue("sender borrow never reached closed-mid-creation teardown", + inTeardown.await(10, TimeUnit.SECONDS)); + awaitCreationWaiter(pool, + "facade close abandoned sender ownership during internal teardown"); + + releaseTeardown.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); + Assert.assertFalse("facade close did not finish", closer.isAlive()); + Assert.assertEquals("internally owned sender must be torn down exactly once", + 1, teardownCount.get()); + Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof LineSenderException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + releaseTeardown.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + private static void awaitBooleanField(Object target, String fieldName) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (field.getBoolean(target)) { + return; + } + Thread.yield(); + } + Assert.fail("field did not become true: " + fieldName); + } + + private static void awaitCreationWaiter(Object pool, String message) throws Exception { + ReentrantLock lock = (ReentrantLock) getField(pool, "lock"); + Condition creationFinished = (Condition) getField(pool, "creationFinished"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + lock.lock(); + try { + if (lock.hasWaiters(creationFinished)) { + return; + } + } finally { + lock.unlock(); + } + Thread.yield(); + } + Assert.fail(message); + } + + private static void awaitOrFail(CountDownLatch latch, String message) { + try { + if (!latch.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException(message); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(message, e); + } + } + + private static Sender fakeSender( + AtomicInteger teardownCount, + CountDownLatch inTeardown, + CountDownLatch releaseTeardown + ) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + if (inTeardown != null) { + inTeardown.countDown(); + awaitOrFail(releaseTeardown, "test never released sender teardown"); + } + if (teardownCount != null) { + teardownCount.incrementAndGet(); + } + return null; + case "toString": + return "FacadeCloseFakeSender"; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + Class returnType = method.getReturnType(); + if (returnType == boolean.class) return false; + if (returnType == byte.class) return (byte) 0; + if (returnType == short.class) return (short) 0; + if (returnType == int.class) return 0; + if (returnType == long.class) return 0L; + if (returnType == float.class) return 0f; + if (returnType == double.class) return 0d; + if (returnType == char.class) return (char) 0; + if (returnType == void.class) return null; + if (returnType.isInstance(proxy)) return proxy; + return null; + } + }); + } + + private static Object getField(Object target, String fieldName) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(target); + } + + private static QuestDBImpl newQuestDB( + int senderMin, + int queryMin, + IntFunction senderFactory, + Consumer connectHook + ) { + return new QuestDBImpl( + SENDER_CFG, QUERY_CFG, + senderMin, 1, + queryMin, 1, + 10_000L, + Long.MAX_VALUE, + Long.MAX_VALUE, + Long.MAX_VALUE, + senderFactory, connectHook); + } +} diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java index b9b0136f..f116a4a6 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java @@ -286,10 +286,17 @@ public void closedMidCreationTeardownRunsOutsideThePoolLock() throws Exception { Assert.assertTrue("borrower never reached the factory", inCreate.await(10, TimeUnit.SECONDS)); - // Close the pool while the creation is in flight: nothing is - // outstanding (the new slot never entered `all`), so this - // returns promptly with `closed` raised. - pool.close(); + // Raise the same early shutdown signal used by QuestDBImpl so + // the creation deterministically takes the teardown branch. + Method markClosing = SenderPool.class.getDeclaredMethod("markClosing"); + markClosing.setAccessible(true); + markClosing.invoke(pool); + + // close() now owns the in-flight creation reservation until + // its closed-mid-creation teardown completes, so run it on a + // separate thread while the creation remains parked. + Thread poolCloser = new Thread(pool::close, "pool-closer"); + poolCloser.start(); // Let the creation finish: the borrower re-locks, observes the // closed pool and starts the delegate teardown, which parks. @@ -313,6 +320,8 @@ public void closedMidCreationTeardownRunsOutsideThePoolLock() throws Exception { lockFree); borrower.join(TimeUnit.SECONDS.toMillis(10)); + poolCloser.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("pool close did not finish", poolCloser.isAlive()); Assert.assertFalse("borrower did not finish", borrower.isAlive()); Assert.assertTrue( "borrow() must surface the closed pool, got: " + borrowOutcome.get(), From 21345b0639edcfbee1464d611935fad4e4402e1a Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Tue, 14 Jul 2026 18:32:41 +0100 Subject: [PATCH 24/26] Harden QWP client recovery and shutdown Protect the durable ACK watermark with redundant CRC records so torn writes cannot skip unacknowledged frames after restart. Bound sender and query pool shutdown waits while retaining late cleanup ownership, make segment recovery cleanup linear and ownership-safe, and count only genuine reconnect sends as replay telemetry. Add deterministic Java 8 regression coverage for crash recovery, pool shutdown, segment-count complexity, cleanup failures, and reconnect metrics. --- .../main/java/io/questdb/client/QuestDB.java | 15 +- .../qwp/client/sf/cursor/AckWatermark.java | 219 +++++++------ .../sf/cursor/CursorWebSocketSendLoop.java | 35 +- .../qwp/client/sf/cursor/SegmentRing.java | 206 +++++++++--- .../questdb/client/impl/QueryClientPool.java | 42 ++- .../io/questdb/client/impl/QuestDBImpl.java | 18 +- .../io/questdb/client/impl/SenderPool.java | 46 ++- .../qwp/client/InitialConnectAsyncTest.java | 163 ++++++++++ .../client/sf/cursor/AckWatermarkTest.java | 41 +++ .../sf/cursor/SegmentRingMembershipTest.java | 209 ++++++++++++ .../qwp/client/sf/cursor/SegmentRingTest.java | 19 +- .../impl/QuestDBImplCloseLifecycleTest.java | 305 +++++++++++++++++- 12 files changed, 1129 insertions(+), 189 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java diff --git a/core/src/main/java/io/questdb/client/QuestDB.java b/core/src/main/java/io/questdb/client/QuestDB.java index 1a9aacae..a4c7cb03 100644 --- a/core/src/main/java/io/questdb/client/QuestDB.java +++ b/core/src/main/java/io/questdb/client/QuestDB.java @@ -130,10 +130,17 @@ static QuestDB connect(CharSequence configurationString) { Sender borrowSender(); /** - * Shuts down the pools, closing every underlying {@link Sender} and - * query client. Idempotent. Threads currently blocked in - * {@link #borrowSender()} or {@link Query#submit()} are released with an - * error. + * Shuts down the pools and their published clients. Idempotent. Threads + * currently blocked in {@link #borrowSender()} or {@link Query#submit()} + * are released with an error. + *

      + * In-progress client creation: close() waits up to the builder's + * {@link QuestDBBuilder#acquireTimeoutMillis(long) acquire timeout}, + * hard-capped at 5 seconds, for a creation blocked in DNS, TCP, TLS, or a + * WebSocket handshake. If that budget expires, the creator retains cleanup + * ownership and closes its unpublished client (and releases any SF slot) + * when construction returns. This keeps close() bounded even when + * {@code connect_timeout} is unset without abandoning late resources. *

      * Outstanding leases: a borrowed {@link Sender} is never torn down * underneath the thread using it. Instead, close() waits up to the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index 625fbf22..c1e6c96a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -24,6 +24,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.std.Crc32c; import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; @@ -43,80 +44,72 @@ * "everything <= N is durable"), so a single monotonic watermark suffices; * no per-frame bitmap is needed. *

      - * Layout (16 bytes, little-endian, mmap'd for the engine's lifetime): + * Layout (128 bytes, little-endian, mmap'd for the engine's lifetime): + * two independently CRC-protected 64-byte records. Each record contains: * 

      - *   offset 0:  u32 magic = 'AKW1' (set once on first write)
- *   offset 4:  u32 reserved (zero)
- *   offset 8:  i64 fsn
+ *   offset 0:   u32 magic = 'AKW1'
+ *   offset 4:   u32 version = 1
+ *   offset 8:   i64 generation
+ *   offset 16:  i64 fsn
+ *   offset 24:  reserved (zero-filled through offset 59)
+ *   offset 60:  u32 CRC32C of bytes [0, 60)
  *
      + * {@link #write(long)} rewrites the record not selected by the current + * generation and stores its CRC last. Recovery selects the valid record with + * the greatest generation. A torn update therefore falls back to the older + * valid record; if neither record validates, recovery conservatively uses the + * segment-derived seed. *

      - * Zero-alloc, store-only ACK writes: {@link #open(String)} opens the - * file and maps the 16 bytes once. {@link #write(long)} is a single - * 8-byte aligned {@code Unsafe.putLong} into the mapped region. Ordinary - * ACK-only manager updates require no malloc/free, read/write syscalls, or - * rename. An 8-byte aligned store is hardware-atomic on x86_64 and - * arm64, and disk-atomic within one sector (the file is 16 bytes — - * trivially within one sector), so a torn FSN across a crash boundary - * is not a concern. - *

      - * Why no CRC: the watermark is an optimization. If the on-disk - * value is somehow corrupted (filesystem bug, hardware fault), the - * recovery path's {@code max(lowestBase - 1, watermark)} clamp absorbs - * the inconsistency: a stale-low watermark just means more re-replay, a - * stale-high watermark is impossible because no write produces an FSN - * higher than the segments on disk could account for. CRC adds - * complexity (multi-store with fence + read-side validate) for - * marginal additional safety. + * Zero-alloc, store-only ACK writes: {@link #open(String)} maps both + * records once. Ordinary ACK-only manager updates mutate the inactive record + * in the mapping and require no malloc/free, read/write syscalls, or rename. *

      * fsync cadence: ordinary ACK-only manager updates call * {@link #write(long)} and stay syscall-free. Each non-empty background disk-trim * quantum calls {@link #sync()} once (one mmap msync and one fd fsync), fsyncs * the slot directory before unlinking, and fsyncs it again after the batch. A - * fully drained close uses the same - * covering order so the durable watermark guards any acknowledged segment that - * a host crash restores. + * fully drained close uses the same covering order so the durable watermark + * guards any acknowledged segment that a host crash restores. *

      - * Lifecycle: single-writer (the {@link SegmentManager} worker - * thread) after construction. Read once at engine startup (any thread, - * before the manager observes the entry). Close releases the mapping - * and fd. Not thread-safe for concurrent writers. + * Lifecycle: single-writer (the {@link SegmentManager} worker thread) + * after construction. Read once at engine startup (any thread, before the + * manager observes the entry). Close releases the mapping and fd. Not + * thread-safe for concurrent writers. */ public final class AckWatermark implements QuietCloseable { /** - * Filename of the watermark within the slot directory. - * Dot-prefixed so directory enumerators that filter by extension - * (segment recovery, OrphanScanner) skip it automatically. + * Filename of the watermark within the slot directory. Dot-prefixed so + * directory enumerators that filter by extension skip it automatically. */ public static final String FILE_NAME = ".ack-watermark"; - public static final int FILE_SIZE = 16; + public static final int FILE_SIZE = 128; /** - * Sentinel returned by {@link #read()} when the watermark file is - * present but has not yet been written to (the magic field is zero - * because the OS zero-filled the freshly created file). + * Sentinel returned by {@link #read()} when neither watermark record is + * valid. */ public static final long INVALID = Long.MIN_VALUE; static final int FILE_MAGIC = 0x31574B41; // 'AKW1' little-endian - private static final int FSN_OFFSET = 8; + private static final int CRC_OFFSET = 60; + private static final int FSN_OFFSET = 16; private static final Logger LOG = LoggerFactory.getLogger(AckWatermark.class); private static final int MAGIC_OFFSET = 0; + private static final int RECORD_SIZE = 64; + private static final int VERSION = 1; private final int fd; private final FilesFacade filesFacade; private final long mmapAddress; private boolean closed; - // Stamped once per process either at open() (if the file already - // has the magic from a prior session) or on the first write() that - // observes it unset. After the flag flips, write() degenerates to - // a single 8-byte putLong against the mapped FSN slot -- no memory - // load of magic on the hot path. Manager-thread-only after - // construction; no synchronisation needed. - private boolean magicWritten; + private long fsn; + private long generation; - private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress, boolean magicAlreadyWritten) { + private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress, + long generation, long fsn) { this.fd = fd; this.filesFacade = filesFacade; + this.fsn = fsn; + this.generation = generation; this.mmapAddress = mmapAddress; - this.magicWritten = magicAlreadyWritten; } @Override @@ -132,13 +125,15 @@ public void close() { } /** - * Opens (creating if absent) the watermark file in {@code slotDir} - * and maps it for the engine's lifetime. Returns {@code null} on - * any setup failure (open fail, mmap fail, unexpected size), leaving - * the caller to choose whether its durability contract permits operation - * without one. Idempotent at the engine layer: a stale file from a - * prior session is reused as-is; the first {@link #write(long)} - * stamps the magic and the new FSN atomically. + * Opens (creating if absent) the watermark file in {@code slotDir} and + * maps it for the engine's lifetime. Returns {@code null} on any setup + * failure, leaving the caller to choose whether its durability contract + * permits operation without one. + *

      + * Wrong-sized files, including the legacy 16-byte non-CRC format, are + * reset. Trusting a legacy FSN would retain the torn-write ambiguity this + * format removes; resetting it causes conservative replay from the + * segment-derived seed instead. */ public static AckWatermark open(String slotDir) { return open(FilesFacade.INSTANCE, slotDir); @@ -146,12 +141,6 @@ public static AckWatermark open(String slotDir) { static AckWatermark open(FilesFacade filesFacade, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; - // Decide by size: existing-and-correct -> openRW preserves the - // previous session's watermark (defeating which is the whole - // point of NOT calling openCleanRW unconditionally); missing or - // wrong-sized -> openCleanRW + allocate creates a fresh - // FILE_SIZE-byte file (zero magic, read() reports INVALID until - // the first write). long existing = filesFacade.exists(filePath) ? filesFacade.length(filePath) : -1L; int fd; if (existing == FILE_SIZE) { @@ -159,8 +148,8 @@ static AckWatermark open(FilesFacade filesFacade, String slotDir) { } else { fd = filesFacade.openCleanRW(filePath); if (fd >= 0 && !filesFacade.allocate(fd, FILE_SIZE)) { - // FilesFacade.allocate contract on a false return: - // close the fd AND unlink the partial file. + // FilesFacade.allocate contract on a false return: close the + // fd and unlink the partial file. filesFacade.close(fd); filesFacade.remove(filePath); fd = -1; @@ -176,19 +165,15 @@ static AckWatermark open(FilesFacade filesFacade, String slotDir) { filesFacade.close(fd); return null; } - // Inspect the existing magic once at open time. If it's already - // set (cross-session reopen, e.g. recovery after a clean - // shutdown), the first write() can skip the magic store - // entirely and degenerate to a single 8-byte FSN put. - int magic = Unsafe.getUnsafe().getInt(addr + MAGIC_OFFSET); - return new AckWatermark(filesFacade, fd, addr, magic == FILE_MAGIC); + Record selected = selectRecord(addr); + return selected == null + ? new AckWatermark(filesFacade, fd, addr, 0L, INVALID) + : new AckWatermark(filesFacade, fd, addr, selected.generation, selected.fsn); } /** - * Best-effort removal of a stale watermark file. Used by the - * engine startup path when no segments are recovered — a stale - * watermark file with no segments behind it is meaningless and - * would only confuse the next session's seed. + * Best-effort removal of a stale watermark file. Used by the engine + * startup path when no segments are recovered. */ public static void removeOrphan(String slotDir) { removeOrphan(FilesFacade.INSTANCE, slotDir); @@ -199,20 +184,20 @@ static boolean removeOrphan(FilesFacade filesFacade, String slotDir) { } /** - * Single-load read of the current FSN. Returns {@link #INVALID} if - * the file has never been written (magic field is zero, i.e. the - * file was freshly created by {@link #open(String)} and no - * {@link #write(long)} has run yet against this slot). + * Returns the FSN from the greatest-generation valid record, or + * {@link #INVALID} when neither record validates. */ public long read() { if (closed) return INVALID; - int magic = Unsafe.getUnsafe().getInt(mmapAddress + MAGIC_OFFSET); - if (magic != FILE_MAGIC) { - // Either freshly created (all zeros) or some kind of corruption. - // Either way, fall back to the segment-derived seed. - return INVALID; + Record selected = selectRecord(mmapAddress); + if (selected == null) { + fsn = INVALID; + generation = 0L; + } else { + fsn = selected.fsn; + generation = selected.generation; } - return Unsafe.getUnsafe().getLong(mmapAddress + FSN_OFFSET); + return fsn; } /** @@ -233,29 +218,65 @@ public void sync() { } /** - * Atomically updates the persisted FSN. Single 8-byte aligned - * store to the mapped region. The first write also stamps the - * magic so the next session's {@link #read()} can distinguish - * "valid watermark" from "freshly created file". + * Updates the inactive record and selects it in memory only after its CRC + * has been stored. The next {@link #sync()} makes the complete update + * durable before any covered segment is deleted. *

      - * Caller responsibility: monotonic ordering. The manager's tick - * loop should only call this when {@code fsn} has advanced past - * the last write. + * Caller responsibility: monotonic ordering. The manager's tick loop only + * calls this when {@code fsn} has advanced past the last write. */ public void write(long fsn) { if (closed) return; - // Steady-state hot path: a single 8-byte aligned putLong, no - // memory load of the magic. Order matters only on the very - // first write of a fresh file: FSN first so the next reader - // that observes the magic (set immediately below) also - // observes a valid FSN -- no fence needed because the same - // thread reads both in program order, and crash recovery - // resumes a fresh process that sees whatever disk-level state - // the kernel flushed. - Unsafe.getUnsafe().putLong(mmapAddress + FSN_OFFSET, fsn); - if (!magicWritten) { - Unsafe.getUnsafe().putInt(mmapAddress + MAGIC_OFFSET, FILE_MAGIC); - magicWritten = true; + long nextGeneration = generation + 1L; + long recordAddress = mmapAddress + (nextGeneration & 1L) * RECORD_SIZE; + Unsafe.getUnsafe().setMemory(recordAddress, RECORD_SIZE, (byte) 0); + Unsafe.getUnsafe().putInt(recordAddress + MAGIC_OFFSET, FILE_MAGIC); + Unsafe.getUnsafe().putInt(recordAddress + 4, VERSION); + Unsafe.getUnsafe().putLong(recordAddress + 8, nextGeneration); + Unsafe.getUnsafe().putLong(recordAddress + FSN_OFFSET, fsn); + int crc = Crc32c.update(Crc32c.INIT, recordAddress, CRC_OFFSET); + Unsafe.getUnsafe().putInt(recordAddress + CRC_OFFSET, crc); + generation = nextGeneration; + this.fsn = fsn; + } + + private static Record readRecord(long address) { + if (Unsafe.getUnsafe().getInt(address + MAGIC_OFFSET) != FILE_MAGIC + || Unsafe.getUnsafe().getInt(address + 4) != VERSION) { + return null; + } + int expectedCrc = Unsafe.getUnsafe().getInt(address + CRC_OFFSET); + int actualCrc = Crc32c.update(Crc32c.INIT, address, CRC_OFFSET); + if (expectedCrc != actualCrc) { + return null; + } + long generation = Unsafe.getUnsafe().getLong(address + 8); + long fsn = Unsafe.getUnsafe().getLong(address + FSN_OFFSET); + if (generation <= 0 || fsn < -1L) { + return null; + } + return new Record(generation, fsn); + } + + private static Record selectRecord(long address) { + Record first = readRecord(address); + Record second = readRecord(address + RECORD_SIZE); + if (first == null) { + return second; + } + if (second == null || first.generation > second.generation) { + return first; + } + return second; + } + + private static final class Record { + private final long fsn; + private final long generation; + + private Record(long generation, long fsn) { + this.fsn = fsn; + this.generation = generation; } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 97b2d7d0..e83d907e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -304,11 +304,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private long nextWireSeq; private volatile SenderProgressDispatcher progressDispatcher; // Frames sent during the post-reconnect catch-up window — i.e. frames - // whose FSN was already published before the wire dropped. A non-zero + // whose FSN completed a send on an earlier connection. A non-zero // value confirms replay is working; a sustained nonzero rate means // the connection is flapping and replay is doing real work each cycle. - // Set at swapClient time to publishedFsn at that moment; cleared back - // to -1 once trySendOne has caught up past it. Used to count replay + // Snapshotted when an established connection enters reconnect; cleared + // back to -1 once trySendOne has caught up past it. Used to count replay // frames without a per-frame branch on the steady-state path. private long replayTargetFsn = -1L; // Recovered orphaned deferred tail: frames [orphanSkipStartFsn .. @@ -1227,6 +1227,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(initial); return; } + snapshotReplayTarget(); LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); @@ -1931,6 +1932,23 @@ private void sendDurableAckKeepaliveIfDue() { } } + /** + * Freezes the highest FSN that completed a send on the connection which + * just failed. Frames published before the first connection, or while a + * reconnect is in progress, have not been sent and therefore do not belong + * to the replay metric. Keep an older, higher target when another failure + * interrupts catch-up so the remaining frames still count as re-sends on + * the following connection. + */ + private void snapshotReplayTarget() { + if (hasEverConnected && nextWireSeq > 0L) { + long lastSentFsn = fsnAtZero + (nextWireSeq - 1L); + if (lastSentFsn > replayTargetFsn) { + replayTargetFsn = lastSentFsn; + } + } + } + /** * Reset wire state for a fresh connection: install the new client, * realign {@code fsnAtZero} to the next unacked FSN, restart wire @@ -1959,11 +1977,12 @@ private void swapClient(WebSocketClient newClient) { long replayStart = engine.ackedFsn() + 1L; this.fsnAtZero = replayStart; this.nextWireSeq = 0L; - // Snapshot publishedFsn at swap time — frames at FSN ≤ this value - // were already on the wire before the drop and will be replayed. - // trySendOne resets replayTargetFsn to -1 once we cross the boundary. - long pubAtSwap = engine.publishedFsn(); - this.replayTargetFsn = pubAtSwap >= replayStart ? pubAtSwap : -1L; + // snapshotReplayTarget froze the completed-send boundary when the + // outage began. ACK progress can move replayStart past that boundary; + // in that case no frame needs re-sending on this connection. + if (replayTargetFsn < replayStart) { + replayTargetFsn = -1L; + } // Drop any durable-ack tracking from the previous connection. The // new connection will re-OK every replayed batch and the server // re-emits cumulative durable-ack watermarks from scratch, so diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 8b8e752c..4dd24b4c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -32,6 +32,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.IdentityHashMap; + /** * Chain of {@link MmapSegment}s presented to the user thread as one logical * append-only log keyed by frame sequence number (FSN). Owns segment @@ -63,6 +65,8 @@ public final class SegmentRing implements QuietCloseable { public static final long BACKPRESSURE_NO_SPARE = -1L; /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */ public static final long PAYLOAD_TOO_LARGE = -2L; + private static final RetainedSegmentMembershipMode DEFAULT_MEMBERSHIP_MODE = + RetainedSegmentMembershipMode.IDENTITY; private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class); // Tally of sealed-list entries inspected by nextSealedAfter(). Test-only // operation count for deterministic traversal-complexity assertions. @@ -176,6 +180,15 @@ public static SegmentRing openExisting(FilesFacade filesFacade, String sfDir, lo * all succeed does it migrate a legacy chain or discard validated spares. */ static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { + return recover(filesFacade, sfDir, maxBytesPerSegment, null); + } + + static Recovery recover( + FilesFacade filesFacade, + String sfDir, + long maxBytesPerSegment, + MembershipObserver membershipObserver + ) { if (!filesFacade.exists(sfDir)) { return Recovery.empty(); } @@ -433,53 +446,73 @@ static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerS } } + // Build membership and clean validated extras while every mmap and + // the manifest still have one local owner. In particular, an OOM + // while allocating the identity map falls through to the outer + // failure cleanup instead of stranding extras after transfer. + RetainedSegmentMembership retained = newDefaultMembership(chain, membershipObserver); + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (!retained.contains(segment)) { + String path = null; + long torn = 0; + Throwable inspectionError = null; + try { + path = segment.path(); + torn = segment.tornTailBytes(); + } catch (Throwable error) { + inspectionError = error; + } + // A close failure is not best-effort: keep local ownership + // and fail through the outer cleanup rather than transfer a + // ring while an extra may still own an mmap or fd. + segment.close(); + all.setQuick(i, null); + if (inspectionError != null) { + warnPostRecoveryCleanupFailure(inspectionError); + continue; + } + try { + cleanupClosedExtra(filesFacade, path, torn); + } catch (Throwable cleanupError) { + warnPostRecoveryCleanupFailure(cleanupError); + } + } + } + quarantineCorrupt(filesFacade, corruptPaths); + for (int i = 1, n = chain.size(); i < n; i++) { chain.get(i - 1).linkSuccessor(chain.get(i)); } SegmentRing ring = new SegmentRing(active, maxBytesPerSegment, manifest); - manifest = null; for (int i = 0, n = chain.size() - 1; i < n; i++) { ring.sealedSegments.add(chain.get(i)); } - // Ownership of the chain transferred. Clean up only validated - // extras; recovery is already successful, so cleanup failure - // must never turn startup into a partially-mutating error or - // orphan the constructed ring -- swallow and let the next - // startup re-examine the leftovers. Extras with a torn tail - // carry evidence of attempted writes -- keep the bytes under a - // .corrupt name instead of unlinking them. - try { - for (int i = 0, n = all.size(); i < n; i++) { - MmapSegment segment = all.get(i); - if (!containsIdentity(chain, segment)) { - String path = segment.path(); - long torn = segment.tornTailBytes(); + // Allocate the return wrapper before transfer. Until this succeeds, + // the outer catch remains the sole owner and can close all segments + // and the manifest if construction or sealed-list growth fails. + Recovery recovery = Recovery.recovered(ring); + manifest = null; + all.clear(); + return recovery; + } catch (Throwable t) { + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment != null) { + try { segment.close(); - if (torn > 0) { - quarantineFile(filesFacade, path); - } else if (!filesFacade.remove(path)) { - LOG.warn("could not remove validated stale/empty SF segment {}", path); - } + } catch (Throwable closeError) { + warnRecoveryCloseFailure(closeError); } } - all.clear(); - quarantineCorrupt(filesFacade, corruptPaths); - } catch (Throwable cleanupError) { - LOG.warn("post-recovery cleanup failed; leftover files will be " - + "re-examined on the next startup", cleanupError); } - return Recovery.recovered(ring); - } catch (Throwable t) { - for (int i = 0, n = all.size(); i < n; i++) { + if (manifest != null) { try { - all.get(i).close(); + manifest.close(); } catch (Throwable closeError) { - LOG.warn("error closing SF segment after recovery failure", closeError); + warnRecoveryCloseFailure(closeError); } } - if (manifest != null) { - manifest.close(); - } throw t; } } @@ -525,13 +558,81 @@ private static MmapSegment chooseEmptyInitial(ObjList all, String s return selected; } + private static void cleanupClosedExtra(FilesFacade filesFacade, String path, long torn) { + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove validated stale/empty SF segment {}", path); + } + } + + private static RetainedSegmentMembership newDefaultMembership( + final ObjList chain, + final MembershipObserver observer + ) { + if (observer != null) { + observer.beforeMembershipAllocation(); + } + if (DEFAULT_MEMBERSHIP_MODE == RetainedSegmentMembershipMode.LINEAR) { + if (observer == null) { + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + for (int i = 0, n = chain.size(); i < n; i++) { + if (chain.get(i) == segment) { + return true; + } + } + return false; + } + }; + } + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + for (int i = 0, n = chain.size(); i < n; i++) { + observer.onMembershipOperation(); + if (chain.get(i) == segment) { + return true; + } + } + return false; + } + }; + } + + final IdentityHashMap retained = new IdentityHashMap<>(chain.size()); + for (int i = 0, n = chain.size(); i < n; i++) { + retained.put(chain.get(i), Boolean.TRUE); + } + if (observer == null) { + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + return retained.containsKey(segment); + } + }; + } + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + observer.onMembershipOperation(); + return retained.containsKey(segment); + } + }; + } + /** Renames every collected corrupt path to {@code .corrupt}, best-effort. */ private static void quarantineCorrupt(FilesFacade filesFacade, ObjList corruptPaths) { if (corruptPaths == null) { return; } for (int i = 0, n = corruptPaths.size(); i < n; i++) { - quarantineFile(filesFacade, corruptPaths.get(i)); + try { + quarantineFile(filesFacade, corruptPaths.get(i)); + } catch (Throwable cleanupError) { + warnPostRecoveryCleanupFailure(cleanupError); + } } } @@ -542,13 +643,6 @@ private static void quarantineFile(FilesFacade filesFacade, String path) { } } - private static boolean containsIdentity(ObjList list, MmapSegment value) { - for (int i = 0, n = list.size(); i < n; i++) { - if (list.get(i) == value) return true; - } - return false; - } - /** * Locates the segment the manifest's {@code activeBase} refers to. * Preference order among same-base candidates: @@ -601,6 +695,29 @@ private static void validateContiguous(ObjList segments) { } } + private static void warnPostRecoveryCleanupFailure(Throwable cleanupError) { + try { + LOG.warn("post-recovery cleanup failed; leftover files will be " + + "re-examined on the next startup", cleanupError); + } catch (Throwable ignored) { + // Cleanup diagnostics must not invalidate a recovered ring. + } + } + + private static void warnRecoveryCloseFailure(Throwable closeError) { + try { + LOG.warn("error closing SF resource after recovery failure", closeError); + } catch (Throwable ignored) { + // Preserve the original recovery failure and continue closing. + } + } + + interface MembershipObserver { + void beforeMembershipAllocation(); + + void onMembershipOperation(); + } + static final class Recovery { private final SegmentRing ring; private final RecoveryStatus status; @@ -632,6 +749,15 @@ enum RecoveryStatus { RECOVERED } + interface RetainedSegmentMembership { + boolean contains(MmapSegment segment); + } + + private enum RetainedSegmentMembershipMode { + IDENTITY, + LINEAR + } + /** * Highest FSN that the server has ACK'd. Read by the segment manager to * decide which sealed segments are safe to munmap + unlink. diff --git a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java index da9249c5..1fc86798 100644 --- a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java +++ b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java @@ -27,6 +27,8 @@ import io.questdb.client.QueryException; import io.questdb.client.cutlass.qwp.client.QwpQueryClient; import org.jetbrains.annotations.TestOnly; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayDeque; import java.util.ArrayList; @@ -56,6 +58,11 @@ public final class QueryClientPool implements AutoCloseable { // close() can never block the caller unbounded. Tunable per pool via // closeQueryTimeoutMillis(long). static final long DEFAULT_CLOSE_QUERY_TIMEOUT_MILLIS = 5_000; + // Hard cap on close()'s wait for a creation blocked in DNS, TCP, TLS, or + // WebSocket setup. A late creator retains the client and native-scratch + // cleanup obligation until it returns and observes the closed pool. + static final long MAX_CLOSE_CREATION_WAIT_MILLIS = 5_000; + private static final Logger LOG = LoggerFactory.getLogger(QueryClientPool.class); private final long acquireTimeoutMillis; private final ArrayList all; private final ArrayDeque available; @@ -319,13 +326,34 @@ public void close() { } closed = true; workerReleased.signalAll(); - // A reservation owns every resource created outside the lock until - // the worker is either published in `all` or fully torn down. No - // user ever received these workers, so close() must not apply an - // abandoned-lease timeout. Preserve interrupts while waiting for - // the internal ownership count to reach zero. - while (inFlightCreations > 0) { - creationFinished.awaitUninterruptibly(); + // A creator can block in DNS, TCP, TLS, WebSocket upgrade, or the + // SERVER_INFO handshake. Wait boundedly rather than turning an + // unset connect_timeout into an unbounded QuestDB.close(). Timing + // out does not abandon the client or its native scratch: the creator + // keeps the reservation and, once construction returns, observes + // closed and shuts the worker down before releasing ownership. + // Preserve interruption while applying the same finite budget. + final long creationWaitMillis = Math.max(0, + Math.min(acquireTimeoutMillis, MAX_CLOSE_CREATION_WAIT_MILLIS)); + final long creationWaitNanos = TimeUnit.MILLISECONDS.toNanos(creationWaitMillis); + final long creationWaitDeadlineNanos = System.nanoTime() + creationWaitNanos; + long creationRemainingNanos = creationWaitNanos; + boolean creationWaitInterrupted = false; + while (inFlightCreations > 0 && creationRemainingNanos > 0) { + try { + creationFinished.awaitNanos(creationRemainingNanos); + } catch (InterruptedException e) { + creationWaitInterrupted = true; + } + creationRemainingNanos = creationWaitDeadlineNanos - System.nanoTime(); + } + if (creationWaitInterrupted) { + Thread.currentThread().interrupt(); + } + if (inFlightCreations > 0) { + LOG.warn("QueryClientPool.close(): {} query client creation(s) still in flight after {}ms; " + + "each creator retains cleanup ownership until construction returns", + inFlightCreations, creationWaitMillis); } snapshot = new ArrayList<>(all); } finally { diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java index 3545d3fa..3d7be5d9 100644 --- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java @@ -181,15 +181,15 @@ public Sender borrowSender() { return senderPool.borrow(); } - // synchronized so concurrent close() callers serialize THROUGH shutdown - // completion, not merely through the `closed` flip. `closed` is set before - // the teardown chain runs, so a plain volatile guard (or a bare CAS) would - // let a second caller observe closed==true and return while the first is - // still inside closeQuietly(senderPool) releasing the flock/mmap/I/O-thread - // resources -- a premature return that breaks the AutoCloseable contract - // that shutdown has completed once close() returns. The monitor makes the - // losing caller block until the winner finishes, then it enters, sees - // `closed` and returns a no-op. No deadlock: the teardown steps + // synchronized so concurrent close() callers serialize THROUGH the bounded + // shutdown sequence, not merely through the `closed` flip. `closed` is set + // before the teardown chain runs, so a plain volatile guard (or a bare CAS) + // would let a second caller observe closed==true and return while the first + // is still closing published resources. The monitor makes the losing caller + // block until the winner finishes, then it enters, sees `closed` and returns + // a no-op. A pool creator that outlives its finite shutdown wait retains and + // eventually cleans its unpublished resources on the creator thread; this + // monitor does not wait past that budget. No deadlock: the teardown steps // (markClosing/housekeeper.stop()/queryPool.close()/senderPool.close()) // never call back into QuestDBImpl.close() on another thread, so nothing // contends for this monitor from within the critical section. diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 32540026..66f09d90 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -96,6 +96,11 @@ public final class SenderPool implements AutoCloseable { // -- the residual window documented on recoverOneSlotStep -- because the // transport has no application-level connect timeout to clamp it. private static final long RECOVERY_DRAIN_BUDGET_MILLIS = 1_000; + // Hard cap on close()'s wait for a creation that is blocked in DNS, TCP, + // TLS, or WebSocket setup. The creator retains ownership after this budget: + // once construction returns, borrow() observes closed and tears down the + // delegate before releasing its reservation (including an SF slot). + static final long MAX_CLOSE_CREATION_WAIT_MILLIS = 5_000; // Hard cap on close()'s outstanding-lease wait. The acquire timeout is a // BORROW policy -- Long.MAX_VALUE legitimately means "block until a slot // frees" -- and must never unbound SHUTDOWN: without this cap a forgotten @@ -937,8 +942,9 @@ void markClosing() { * free their native memory under its feet -- a use-after-free / SEGV, not * an exception (C1). Instead: *

        - *
      1. waits for every internally owned creation to publish or complete its - * closed-mid-creation teardown; then
        2. + *
      2. waits boundedly for internally owned creations; a late creator keeps + * ownership and performs its closed-mid-creation teardown asynchronously; + * then
        3. *
      3. waits boundedly (up to {@code acquireTimeoutMillis}, hard-capped at * {@link #MAX_CLOSE_LEASE_WAIT_MILLIS}) for outstanding leases to come home -- {@link #giveBack} and {@link #discardBroken} * observe {@code closed} and tear each delegate down on the returning @@ -965,14 +971,34 @@ public void close() { closed = true; // Wake parked borrowers so they observe the shutdown and throw. slotReleased.signalAll(); - // A creation reservation represents pool-owned resources from the - // moment borrow() leaves the lock through either publication or - // completed cleanup. Unlike an abandoned user-visible lease, this - // ownership cannot be leaked after close() returns, so wait without - // a timeout. awaitUninterruptibly preserves the interrupt flag while - // maintaining the shutdown ownership invariant. - while (inFlightCreations > 0) { - creationFinished.awaitUninterruptibly(); + // A creator can block in DNS, TCP, TLS, or WebSocket setup. Wait + // boundedly rather than turning an unset connect_timeout into an + // unbounded QuestDB.close(). Timing out does not abandon ownership: + // the reservation and any SF slot stay assigned to the creator, + // which observes closed and tears down a late result before releasing + // them. Preserve the caller's interrupt status while still applying + // the same finite wait budget. + final long creationWaitMillis = Math.max(0, + Math.min(acquireTimeoutMillis, MAX_CLOSE_CREATION_WAIT_MILLIS)); + final long creationWaitNanos = TimeUnit.MILLISECONDS.toNanos(creationWaitMillis); + final long creationWaitDeadlineNanos = System.nanoTime() + creationWaitNanos; + long creationRemainingNanos = creationWaitNanos; + boolean creationWaitInterrupted = false; + while (inFlightCreations > 0 && creationRemainingNanos > 0) { + try { + creationFinished.awaitNanos(creationRemainingNanos); + } catch (InterruptedException e) { + creationWaitInterrupted = true; + } + creationRemainingNanos = creationWaitDeadlineNanos - System.nanoTime(); + } + if (creationWaitInterrupted) { + Thread.currentThread().interrupt(); + } + if (inFlightCreations > 0) { + LOG.warn("SenderPool.close(): {} sender creation(s) still in flight after {}ms; " + + "each creator retains cleanup ownership and releases its SF slot when construction returns", + inFlightCreations, creationWaitMillis); } // Bounded graceful wait for outstanding leases. A slot is borrowed // iff it is in `all` but not in `available`; retireLease's diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java index 173374ec..5f04ca68 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java @@ -27,6 +27,7 @@ import io.questdb.client.Sender; import io.questdb.client.SenderError; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.jetbrains.annotations.NotNull; import org.junit.Assert; @@ -43,8 +44,10 @@ import java.nio.charset.StandardCharsets; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; /** * Behavior of {@code initial_connect_retry=async}: the producer-thread @@ -205,6 +208,78 @@ public void testAsyncDeliversBufferedRowsWhenServerArrivesLate() throws Exceptio } } + @Test + public void testAsyncMetricsDistinguishInitialSendFromReconnectReplay() throws Exception { + ReplayMetricsHandler handler = new ReplayMetricsHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + String cfg = "ws::addr=localhost:" + server.getPort() + + sfDirOpt() + ";initial_connect_retry=async" + + ";reconnect_initial_backoff_millis=20" + + ";reconnect_max_backoff_millis=200" + + ";close_flush_timeout_millis=2000;"; + Sender sender = Sender.fromConfig(cfg); + try { + QwpWebSocketSender wss = (QwpWebSocketSender) sender; + handler.bind(wss, server); + + // Publish before the server starts accepting. The frame has + // never been on any wire, so its eventual first delivery must + // increase sent, but not replayed. + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + awaitAtLeastOneConnectAttempt(wss); + Assert.assertEquals("the server must not handshake before start", 0, server.handshakeCount()); + + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("the initial frame must be ACKed", + handler.awaitInitialAck(5, TimeUnit.SECONDS)); + awaitMetricAtLeast("initial ACK", wss::getTotalAcks, 1L); + Assert.assertEquals("exactly one frame must be sent initially", + 1L, wss.getTotalFramesSent()); + long replayedAfterInitialDelivery = wss.getTotalFramesReplayed(); + + // The second frame reaches the established connection, but the + // fixture closes that connection without ACKing it. It must then + // arrive again on a genuinely new connection and count once as + // replayed. + sender.table("foo").longColumn("v", 2L).atNow(); + sender.flush(); + Assert.assertTrue("the fixture must close after the unacked frame", + handler.awaitDisconnect(5, TimeUnit.SECONDS)); + Assert.assertTrue("the reconnect must reach the gated server", + server.awaitRoleReject(5, TimeUnit.SECONDS)); + + // Publish another frame while reconnect is blocked at a role + // reject, then reopen the gate. This frame has never been sent + // and must not inflate the replay counter after reconnect. + sender.table("foo").longColumn("v", 3L).atNow(); + sender.flush(); + server.setRejectWithRole(null); + + Assert.assertTrue("the unacked frame must be replayed", + handler.awaitReplay(5, TimeUnit.SECONDS)); + Assert.assertTrue("the outage-queued frame must be delivered", + handler.awaitOutageQueuedDelivery(5, TimeUnit.SECONDS)); + awaitMetricAtLeast("replayed frame", wss::getTotalFramesReplayed, 1L); + awaitMetricAtLeast("post-reconnect ACKs", wss::getTotalAcks, 3L); + + Assert.assertTrue("replay must arrive on a new WebSocket connection", + handler.wasReplayOnNewConnection()); + Assert.assertTrue("the server must observe a reconnect", + server.handshakeCount() >= 2); + Assert.assertEquals("initial delivery is not a replay", + 0L, replayedAfterInitialDelivery); + Assert.assertEquals("three first sends plus one genuine resend", + 4L, wss.getTotalFramesSent()); + Assert.assertEquals("only the genuine resend is replayed", + 1L, wss.getTotalFramesReplayed()); + } finally { + closeQuietly(sender); + } + } + } + @Test public void testAsyncReturnsImmediatelyWithNoServer() { // No server. With async mode, fromConfig must return fast — the @@ -345,6 +420,17 @@ private static void awaitAtLeastOneConnectAttempt(QwpWebSocketSender wss) { * before the budget expired would satisfy it even if the loop then * exited. */ + private static void awaitMetricAtLeast(String label, LongSupplier metric, long expected) { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (metric.getAsLong() < expected) { + if (System.nanoTime() > deadlineNanos) { + throw new AssertionError(label + " did not reach " + expected + + " within 5s; current=" + metric.getAsLong()); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + private static void awaitReconnectAttemptsAdvance(QwpWebSocketSender wss) { long snapshot = wss.getTotalReconnectAttempts(); long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); @@ -476,6 +562,83 @@ public void onError(@NotNull SenderError err) { } } + private static class ReplayMetricsHandler implements TestWebSocketServer.WebSocketServerHandler { + private final CountDownLatch disconnect = new CountDownLatch(1); + private final CountDownLatch initialAck = new CountDownLatch(1); + private final CountDownLatch outageQueuedDelivery = new CountDownLatch(1); + private final AtomicLong received = new AtomicLong(); + private final CountDownLatch replay = new CountDownLatch(1); + private final AtomicBoolean replayOnNewConnection = new AtomicBoolean(); + private TestWebSocketServer.ClientHandler initialClient; + private volatile TestWebSocketServer server; + private volatile QwpWebSocketSender sender; + + boolean awaitDisconnect(long timeout, TimeUnit unit) throws InterruptedException { + return disconnect.await(timeout, unit); + } + + boolean awaitInitialAck(long timeout, TimeUnit unit) throws InterruptedException { + return initialAck.await(timeout, unit); + } + + boolean awaitOutageQueuedDelivery(long timeout, TimeUnit unit) throws InterruptedException { + return outageQueuedDelivery.await(timeout, unit); + } + + boolean awaitReplay(long timeout, TimeUnit unit) throws InterruptedException { + return replay.await(timeout, unit); + } + + void bind(QwpWebSocketSender sender, TestWebSocketServer server) { + this.sender = sender; + this.server = server; + } + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + long ordinal = received.incrementAndGet(); + try { + if (ordinal == 1L) { + initialClient = client; + client.sendBinary(AckHandler.buildAck(0L)); + initialAck.countDown(); + } else if (ordinal == 2L) { + // Receipt alone can race the sender's post-send metric + // increments. Wait until sendBinary returned before closing, + // proving this frame completed a send and must be replayed. + awaitSentFrames(2L); + server.setRejectWithRole("REPLICA"); + client.sendClose(WebSocketCloseCode.GOING_AWAY, "test reconnect"); + disconnect.countDown(); + } else if (ordinal == 3L) { + replayOnNewConnection.set(client != initialClient); + client.sendBinary(AckHandler.buildAck(0L)); + replay.countDown(); + } else if (ordinal == 4L) { + client.sendBinary(AckHandler.buildAck(1L)); + outageQueuedDelivery.countDown(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + boolean wasReplayOnNewConnection() { + return replayOnNewConnection.get(); + } + + private void awaitSentFrames(long expected) { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (sender.getTotalFramesSent() < expected) { + if (System.nanoTime() > deadlineNanos) { + throw new AssertionError("sent-frame metric did not reach " + expected + + " before disconnect; current=" + sender.getTotalFramesSent()); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + } + /** * Raw-socket fixture: every accepted connection responds with HTTP * 401 Unauthorized and closes. Used to drive the async-init diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java index 001f62ce..27bef939 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.After; import org.junit.Before; @@ -76,6 +78,45 @@ public void testCrossSessionPersistence() throws Exception { }); } + @Test + public void testFallsBackFromTornPlausibleHighValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (AckWatermark watermark = AckWatermark.open(slotDir)) { + assertNotNull(watermark); + watermark.write(255L); + watermark.write(256L); + watermark.sync(); + } + + // Model a torn little-endian 255 -> 256 transition: the high byte + // reached storage while the low byte retained 0xff, yielding the + // false FSN 511. A recovered ring with publishedFsn=599 cannot + // reject that value using its segment ceiling. + long publishedFsn = 599L; + long corruptFsn = 511L; + assertTrue(corruptFsn <= publishedFsn); + String path = slotDir + "/" + AckWatermark.FILE_NAME; + int fd = Files.openRW(path); + assertTrue(fd >= 0); + long corruptByte = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(corruptByte, (byte) 0xff); + long fsnOffset = AckWatermark.FILE_SIZE == 16 ? 8L : 16L; + assertEquals(1L, Files.write(fd, corruptByte, 1, fsnOffset)); + assertEquals(0, Files.fsync(fd)); + } finally { + Files.close(fd); + Unsafe.free(corruptByte, 1, MemoryTag.NATIVE_DEFAULT); + } + + try (AckWatermark watermark = AckWatermark.open(slotDir)) { + assertNotNull(watermark); + assertEquals("torn latest record must fall back to the older watermark", + 255L, watermark.read()); + } + }); + } + @Test public void testFreshFileReadsAsInvalid() throws Exception { // open() creates the file zero-filled, so magic is 0 and read() diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java new file mode 100644 index 00000000..d8813b7f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java @@ -0,0 +1,209 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class SegmentRingMembershipTest { + + private static final String CURSOR_PACKAGE = "io.questdb.client.cutlass.qwp.client.sf.cursor."; + private static final long SEGMENT_SIZE = MmapSegment.HEADER_SIZE + + MmapSegment.FRAME_HEADER_SIZE + 16; + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-ring-membership-" + System.nanoTime()).toString(); + assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir == null) { + return; + } + long find = Files.findFirst(tmpDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(tmpDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(tmpDir); + } + + @Test + public void testLargeRecoveryCountsMembershipPrimitives() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int n = 2048; + createChain(n); + + CountingOperations operations = new CountingOperations(); + try (SegmentRing ring = recover(newMembershipObserver(operations, false))) { + assertRecoveredChain(ring, n); + } + long linearBound = 2L * n; + assertTrue("production default took " + operations.count + + " membership operations; expected fewer than " + linearBound, + operations.count < linearBound); + assertEquals("production default identity map must do one lookup per discovered segment", + n, operations.count); + }); + } + + @Test + public void testMembershipAllocationFailureRetainsLocalOwnership() throws Exception { + TestUtils.assertMemoryLeak(() -> { + createChain(8); + String extraPath = tmpDir + "/sf-extra.sfa"; + MmapSegment.create(extraPath, 8, SEGMENT_SIZE).close(); + try { + recover(newMembershipObserver(null, true)); + fail("expected injected membership allocation failure"); + } catch (OutOfMemoryError expected) { + assertEquals("injected membership allocation failure", expected.getMessage()); + } + + try (SegmentRing ring = SegmentRing.openExisting( + FilesFacade.INSTANCE, + tmpDir, + SEGMENT_SIZE + )) { + assertRecoveredChain(ring, 8); + } + assertFalse("retry must close and remove the unselected empty extra", Files.exists(extraPath)); + }); + } + + private static void assertRecoveredChain(SegmentRing ring, int n) { + assertNotNull(ring); + assertEquals(n, ring.nextSeqHint()); + assertEquals(n - 1, ring.publishedFsn()); + assertEquals(n - 1, ring.getSealedSegments().size()); + } + + private void createChain(int n) { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 16; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) i); + } + for (int i = 0; i < n; i++) { + String name = String.format("sf-%05d.sfa", i); + try (MmapSegment segment = MmapSegment.create(tmpDir + "/" + name, i, SEGMENT_SIZE)) { + assertTrue("setup append " + i, segment.tryAppend(buf, 16) >= 0); + } + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + } + + private SegmentRing recover(Object membershipObserver) throws Exception { + Class observerClass = Class.forName(CURSOR_PACKAGE + "SegmentRing$MembershipObserver"); + Method recover = SegmentRing.class.getDeclaredMethod( + "recover", + FilesFacade.class, + String.class, + long.class, + observerClass + ); + recover.setAccessible(true); + Object recovery; + try { + recovery = recover.invoke(null, FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE, membershipObserver); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Error) { + throw (Error) cause; + } + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + throw new AssertionError(cause); + } + Method ring = recovery.getClass().getDeclaredMethod("ring"); + ring.setAccessible(true); + return (SegmentRing) ring.invoke(recovery); + } + + private static Object newMembershipObserver( + final CountingOperations operations, + final boolean failAllocation + ) throws Exception { + Class observerClass = Class.forName(CURSOR_PACKAGE + "SegmentRing$MembershipObserver"); + return Proxy.newProxyInstance( + SegmentRing.class.getClassLoader(), + new Class[]{observerClass}, + (proxy, method, args) -> { + if ("beforeMembershipAllocation".equals(method.getName())) { + if (failAllocation) { + throw new OutOfMemoryError("injected membership allocation failure"); + } + } else if ("onMembershipOperation".equals(method.getName())) { + operations.onOperation(); + } + return null; + } + ); + } + + private static final class CountingOperations { + private long count; + + private void onOperation() { + count++; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index c6466de7..9bc08a6a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -616,22 +616,19 @@ public void testNextSealedAfterSurvivesConcurrentRotationAndHeadTrim() throws Ex /** * Open-time sort regression: at the documented {@code sf_max_total_bytes - * / sf_max_bytes} ceiling (~16K segments) an O(N²) sort over the - * recovered segments burns multi-second wall time before the I/O thread - * can start. The previous selection-sort implementation regressed an - * earlier perf fix on the legacy {@code SegmentLog} path; this test - * guards the cursor path against the same regression. + * / sf_max_bytes} ceiling (~16K segments), an O(N²) sort over the + * recovered segments delays the I/O thread. This test guards the sort + * with a deterministic comparison count. *

        * Constructs N=2048 valid one-frame segments with names assigned in * lexicographic order — the exact pattern {@code readdir} produces on * many filesystems (and the worst case for a naive first-element pivot). * Recovers, asserts contiguous baseSeq ordering and total frame count, - * and bounds wall time at 5 s. With the median-of-three quicksort the - * test completes in well under a second; an O(N²) regression at this - * scale climbs back into multi-second territory. + * and bounds sort comparisons independently of CI timing or filesystem + * throughput. */ @Test - public void testLargeSegmentCountReopensInOrder() throws Exception { + public void testLargeSegmentCountRecoveryIsLogLinear() throws Exception { TestUtils.assertMemoryLeak(() -> { final int n = 2048; long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); @@ -663,6 +660,8 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { n, ring.nextSeqHint()); // publishedFsn = n - 1 (last frame visible). assertEquals(n - 1, ring.publishedFsn()); + assertEquals("full valid chain must retain every segment", + n - 1, ring.getSealedSegments().size()); // O(N log N) quicksort with good pivots does ~2-3 * N * log2(N) // comparisons; the partition-pass + median-of-three counter we // increment per recursive frame upper-bounds this at roughly @@ -688,7 +687,7 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { /** * Adversarial-order companion to - * {@link #testLargeSegmentCountReopensInOrder}: that test covers the + * {@link #testLargeSegmentCountRecoveryIsLogLinear}: that test covers the * already-sorted readdir order median-of-three was chosen for; this one * covers the orders median-of-three does NOT defend. On the * pre-introsort quicksort, exact simulation at N=16384 measured ~22.6M diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java index dd697049..65e57f85 100644 --- a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java @@ -31,6 +31,8 @@ import io.questdb.client.impl.QueryClientPool; import io.questdb.client.impl.QuestDBImpl; import io.questdb.client.impl.SenderPool; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -52,6 +54,294 @@ public class QuestDBImplCloseLifecycleTest { private static final String QUERY_CFG = "ws::addr=127.0.0.1:9000;"; private static final String SENDER_CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + @Test(timeout = 30_000) + public void facadeCloseIsBoundedByZeroAcquireTimeoutDuringQueryCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + Consumer connectHook = client -> { + client.setBeforeCloseHookForTest(teardownCount::incrementAndGet); + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released query creation"); + }; + QuestDBImpl db = newQuestDB( + SENDER_CFG, 0, 0, 0, slotIndex -> fakeSender(null, null, null), connectHook); + QueryClientPool pool = (QueryClientPool) getField(db, "queryPool"); + AtomicReference borrowOutcome = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + db.borrowQuery(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "bounded-query-borrower"); + Thread closer = new Thread(db::close, "bounded-query-closer"); + long nativeBaseline = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT); + try { + borrower.start(); + Assert.assertTrue("query borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, pool.inFlightCreations()); + + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "facade close exceeded its zero creation-wait budget", + closer.isAlive()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, pool.inFlightCreations()); + Assert.assertEquals("the still-constructing query client must remain live", 0, teardownCount.get()); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("query borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late query creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late query creation reservation must be released", 0, pool.inFlightCreations()); + Assert.assertEquals("late query cleanup must release native scratch", + nativeBaseline, Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT)); + Assert.assertTrue("borrowQuery() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof QueryException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseIsBoundedByZeroAcquireTimeoutDuringSenderCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + IntFunction senderFactory = slotIndex -> { + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released sender creation"); + return fakeSender(teardownCount, null, null); + }; + String senderConfig = "ws::addr=localhost:1;sf_dir=" + + System.getProperty("java.io.tmpdir") + "/qdb-bounded-pool-" + System.nanoTime() + ";"; + QuestDBImpl db = newQuestDB(senderConfig, 0, 0, 0, senderFactory, client -> { + }); + SenderPool pool = (SenderPool) getField(db, "senderPool"); + AtomicReference borrowOutcome = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + db.borrowSender(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "bounded-sender-borrower"); + Thread closer = new Thread(db::close, "bounded-sender-closer"); + try { + borrower.start(); + Assert.assertTrue("sender borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertTrue("SF slot must stay reserved during creation", + ((boolean[]) getField(pool, "slotInUse"))[0]); + + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "facade close exceeded its zero creation-wait budget", + closer.isAlive()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertTrue("close must not abandon the reserved SF slot", + ((boolean[]) getField(pool, "slotInUse"))[0]); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late sender creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late sender creation reservation must be released", + 0, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertFalse("late sender cleanup must release the SF slot", + ((boolean[]) getField(pool, "slotInUse"))[0]); + Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof LineSenderException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseIsBoundedUnderRepeatedInterruptsDuringQueryCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + Consumer connectHook = client -> { + client.setBeforeCloseHookForTest(teardownCount::incrementAndGet); + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released query creation"); + }; + QuestDBImpl db = newQuestDB( + SENDER_CFG, 0, 0, 100, slotIndex -> fakeSender(null, null, null), connectHook); + QueryClientPool pool = (QueryClientPool) getField(db, "queryPool"); + AtomicReference borrowOutcome = new AtomicReference<>(); + AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); + AtomicBoolean keepInterrupting = new AtomicBoolean(true); + AtomicInteger interruptCount = new AtomicInteger(); + Thread borrower = new Thread(() -> { + try { + db.borrowQuery(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "interrupted-query-borrower"); + Thread closer = new Thread(() -> { + db.close(); + closeReturnedInterrupted.set(Thread.currentThread().isInterrupted()); + }, "interrupted-query-closer"); + Thread interrupter = new Thread(() -> { + while (keepInterrupting.get()) { + interruptCount.incrementAndGet(); + closer.interrupt(); + Thread.yield(); + } + }, "query-close-interrupter"); + long nativeBaseline = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT); + try { + borrower.start(); + Assert.assertTrue("query borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, pool.inFlightCreations()); + + closer.start(); + awaitCreationWaiter(pool, + "facade close did not wait while query construction was internally owned"); + interrupter.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "repeated interrupts restarted the query creation-wait deadline", + closer.isAlive()); + Assert.assertTrue("test did not repeatedly interrupt query close", interruptCount.get() > 1); + Assert.assertTrue("facade close must restore query closer interruption", + closeReturnedInterrupted.get()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, pool.inFlightCreations()); + Assert.assertEquals("the still-constructing query client must remain live", 0, teardownCount.get()); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("query borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late query creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late query creation reservation must be released", 0, pool.inFlightCreations()); + Assert.assertEquals("late query cleanup must release native scratch", + nativeBaseline, Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT)); + Assert.assertTrue("borrowQuery() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof QueryException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + keepInterrupting.set(false); + releaseCreation.countDown(); + interrupter.join(TimeUnit.SECONDS.toMillis(10)); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseIsBoundedUnderRepeatedInterruptsDuringSenderCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + IntFunction senderFactory = slotIndex -> { + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released sender creation"); + return fakeSender(teardownCount, null, null); + }; + String senderConfig = "ws::addr=localhost:1;sf_dir=" + + System.getProperty("java.io.tmpdir") + "/qdb-interrupted-pool-" + System.nanoTime() + ";"; + QuestDBImpl db = newQuestDB(senderConfig, 0, 0, 100, senderFactory, client -> { + }); + SenderPool pool = (SenderPool) getField(db, "senderPool"); + AtomicReference borrowOutcome = new AtomicReference<>(); + AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); + AtomicBoolean keepInterrupting = new AtomicBoolean(true); + AtomicInteger interruptCount = new AtomicInteger(); + Thread borrower = new Thread(() -> { + try { + db.borrowSender(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "interrupted-sender-borrower"); + Thread closer = new Thread(() -> { + db.close(); + closeReturnedInterrupted.set(Thread.currentThread().isInterrupted()); + }, "interrupted-sender-closer"); + Thread interrupter = new Thread(() -> { + while (keepInterrupting.get()) { + interruptCount.incrementAndGet(); + closer.interrupt(); + Thread.yield(); + } + }, "sender-close-interrupter"); + try { + borrower.start(); + Assert.assertTrue("sender borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertTrue("SF slot must stay reserved during creation", + ((boolean[]) getField(pool, "slotInUse"))[0]); + + closer.start(); + awaitCreationWaiter(pool, + "facade close did not wait while sender construction was internally owned"); + interrupter.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "repeated interrupts restarted the sender creation-wait deadline", + closer.isAlive()); + Assert.assertTrue("test did not repeatedly interrupt sender close", interruptCount.get() > 1); + Assert.assertTrue("facade close must restore sender closer interruption", + closeReturnedInterrupted.get()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertTrue("close must not abandon the reserved SF slot", + ((boolean[]) getField(pool, "slotInUse"))[0]); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late sender creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late sender creation reservation must be released", + 0, ((Integer) getField(pool, "inFlightCreations")).intValue()); + Assert.assertFalse("late sender cleanup must release the SF slot", + ((boolean[]) getField(pool, "slotInUse"))[0]); + Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof LineSenderException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + keepInterrupting.set(false); + releaseCreation.countDown(); + interrupter.join(TimeUnit.SECONDS.toMillis(10)); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + @Test(timeout = 30_000) public void facadeCloseWaitsForQueryCreationAndTeardown() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -281,12 +571,23 @@ private static QuestDBImpl newQuestDB( int queryMin, IntFunction senderFactory, Consumer connectHook + ) { + return newQuestDB(SENDER_CFG, senderMin, queryMin, 10_000L, senderFactory, connectHook); + } + + private static QuestDBImpl newQuestDB( + String senderConfig, + int senderMin, + int queryMin, + long acquireTimeoutMillis, + IntFunction senderFactory, + Consumer connectHook ) { return new QuestDBImpl( - SENDER_CFG, QUERY_CFG, + senderConfig, QUERY_CFG, senderMin, 1, queryMin, 1, - 10_000L, + acquireTimeoutMillis, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, From 94a0ae6b72e31f30ace12798a631ca2efd3dabbc Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Tue, 14 Jul 2026 19:26:48 +0100 Subject: [PATCH 25/26] Isolate flock retry ordering test EngineClosePublishAfterFlockReleaseTest started the shared retry driver after injecting an invalid slot-lock descriptor. It then returned before the driver's initial park expired. A following test could observe the global driver as active and fail, as Windows CI did. Inject a synchronous retry-driver start failure for this explicit-close retry test. Restore the factory in a nested finally block. Dedicated retry-driver tests continue to cover the asynchronous lifecycle. --- ...gineClosePublishAfterFlockReleaseTest.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java index c142dd6a..0c28ea16 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -200,6 +200,15 @@ public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws E fdField.setAccessible(true); int realFd = fdField.getInt(slotLock); assertTrue("precondition: live flock fd", realFd >= 0); + + // This test owns the explicit close() retry. Prevent the shared + // asynchronous retry driver from racing it and from surviving into + // another test class; FlockReleaseRetryDriverTest covers the real + // driver lifecycle separately. A start failure synchronously clears + // the shared retry queue and resets the engine's scheduling flag. + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + throw new IllegalStateException("retry driver disabled for explicit close retry test"); + }); try { // A non-negative fd no process has open: close(2) fails EBADF, // so finishClose's release() confirmation fails. @@ -230,11 +239,15 @@ public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws E // good — eventual completion implies reusable capacity. } } finally { - // If an assertion failed before the successful retry, restore - // and release the real fd so the test never leaks a flock. - if (!engine.isCloseCompleted()) { - fdField.setInt(slotLock, realFd); - assertTrue("restored fd must release cleanly", slotLock.release()); + try { + // If an assertion failed before the successful retry, restore + // and release the real fd so the test never leaks a flock. + if (!engine.isCloseCompleted()) { + fdField.setInt(slotLock, realFd); + assertTrue("restored fd must release cleanly", slotLock.release()); + } + } finally { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); } } }); From d5673b3ffa5c66e3bb0d92bcf3191a4471ada9b1 Mon Sep 17 00:00:00 2001 From: Vlad Ilyushchenko Date: Tue, 14 Jul 2026 20:50:15 +0100 Subject: [PATCH 26/26] Continue direct pool startup recovery Direct SenderPool instances now continue store-and-forward recovery after a transient failure or startup budget exhaustion. The recovery driver uses elapsed-time budget accounting, throttles retries, and shuts down before delegate teardown. Guard driver startup so constructor failures quiesce the driver and close prewarmed delegates. Add deterministic recovery, lifecycle, cleanup, and mutation-sensitive tests. --- .../io/questdb/client/impl/SenderPool.java | 321 ++++++++--- .../client/test/impl/SenderPoolSfTest.java | 514 +++++++++++++++++- 2 files changed, 752 insertions(+), 83 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 66f09d90..49cac21d 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -40,6 +40,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Iterator; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; @@ -85,8 +86,8 @@ public final class SenderPool implements AutoCloseable { // Per-slot wall-clock cap on a single startup-recovery drain. Kept BELOW the // PoolHousekeeper stop/join budget (PoolHousekeeper.STOP_TIMEOUT_MILLIS) so a // recovery drain still in flight when close() arrives cannot outlive the - // housekeeper join -- the residual-budget bound that, together with the - // early markClosing() signal, keeps close() prompt (C1 fix). + // driver join -- the residual-budget bound that, together with the early + // markClosing() signal, keeps close() prompt (C1 fix). // // This caps only the DRAIN. The recovery build that precedes it is bounded // separately: recovery delegates force initial_connect_mode=OFF (see @@ -96,6 +97,10 @@ public final class SenderPool implements AutoCloseable { // -- the residual window documented on recoverOneSlotStep -- because the // transport has no application-level connect timeout to clamp it. private static final long RECOVERY_DRAIN_BUDGET_MILLIS = 1_000; + // A direct SenderPool has no PoolHousekeeper to retry a transient startup + // recovery failure. Its private driver waits this long between failed + // attempts so an unavailable server does not cause a hot retry loop. + private static final long RECOVERY_RETRY_INTERVAL_MILLIS = 1_000; // Hard cap on close()'s wait for a creation that is blocked in DNS, TCP, // TLS, or WebSocket setup. The creator retains ownership after this budget: // once construction returns, borrow() observes closed and tears down the @@ -111,6 +116,11 @@ public final class SenderPool implements AutoCloseable { private final long acquireTimeoutMillis; private final ArrayList all; private final ArrayDeque available; + // Test-only constructor seam. Runs immediately before a possibly-started + // recovery driver is joined after constructor failure. Null in production; + // lifecycle tests use it to release a deliberately held driver and prove + // delegate cleanup happens only after the join. + private final Runnable beforeFailedStartupRecoveryJoinHook; private final String configurationString; // Signals completion of internally owned creation lifecycles. Kept // separate from slotReleased so a capacity waiter cannot consume the only @@ -134,13 +144,13 @@ public final class SenderPool implements AutoCloseable { private final Runnable postFactoryHook; // Factory for startup-recovery delegates. Distinct from senderFactory so a // recoverer can force a non-blocking initial connect (initial_connect_mode= - // OFF) regardless of user config: a recovery build runs on the - // PoolHousekeeper thread and must NOT inherit SYNC (auto-enabled by any - // reconnect_* knob), which would retry the connect for the whole reconnect - // budget inside build() -- far past PoolHousekeeper.STOP_TIMEOUT_MILLIS, so - // a close() landing during that build would make housekeeper.stop()'s join - // time out and leave the recoverer holding the slot flock after close() - // returned (M1). Mirrors senderFactory's test seam: an injected factory + // OFF) regardless of user config: a recovery build runs on a private direct + // driver or the PoolHousekeeper thread and must NOT inherit SYNC + // (auto-enabled by any reconnect_* knob), which would retry the connect for + // the whole reconnect budget inside build() -- far past + // PoolHousekeeper.STOP_TIMEOUT_MILLIS, so a close() landing during that + // build could make its driver join time out and leave the recoverer holding + // the slot flock after close() returned (M1). Mirrors senderFactory's test seam: an injected factory // (non-null) drives BOTH paths so white-box recovery tests keep control. private final IntFunction recoverySenderFactory; private final ReentrantLock lock = new ReentrantLock(); @@ -161,11 +171,28 @@ public final class SenderPool implements AutoCloseable { private final Condition slotReleased; // True iff the configuration enables store-and-forward (sf_dir set). private final boolean storeAndForward; + // Direct pools run one inline recovery attempt for backwards-compatible + // startup behavior, then this daemon continues any backlog left by budget + // exhaustion or a transient failure. Deferred pools use PoolHousekeeper and + // leave this null. + private final Object startupRecoverySignal = new Object(); + private final Thread startupRecoveryThread; + // Test-only constructor seam for the failed-retry operation. Production + // uses waitForStartupRecoveryRetry(), which performs the one-second signal + // wait; lifecycle tests inject an event barrier without wall-clock checks. + private final Runnable startupRecoveryWaiter; + // Test seam: runs after the direct recovery driver's bounded join returns. + // Null in production; lifecycle tests prove the joined thread is quiescent. + private volatile Runnable afterStartupRecoveryJoinHook; // Test seam: runs immediately before a capacity-starved borrow enters its // condition wait, while it still holds the pool lock. Null in production; // concurrency tests use a latch here to prove that several borrowers have // all reached the wait path before recovering retired capacity. private volatile Runnable beforeBorrowWaitHook; + // Test seam: runs immediately before the direct recovery driver's bounded + // join. Null in production; lifecycle tests coordinate close() with a + // deliberately held driver. + private volatile Runnable beforeStartupRecoveryJoinHook; // Test seam: runs after a capacity-starved borrow's condition wait has // exhausted its positive timeout, before the loop's terminal pass. Null in // production; regression tests release a retired slot here to prove that @@ -223,14 +250,15 @@ public final class SenderPool implements AutoCloseable { // (recoverOneSlotStep): each is reserved under `lock` for the // duration of its drain and counted in the borrow() cap check so a // concurrent borrow can neither over-allocate past maxSize nor target a - // dir being recovered. Only ever non-zero on the deferred (housekeeper- - // driven) recovery path, where recovery overlaps borrow()/return; on the - // inline construction path the pool is still single-threaded. Guarded by - // lock; only ever ticks for SF slots. + // dir being recovered. The inline constructor drive is single-threaded, + // but the continuing direct-pool driver and deferred housekeeper driver can + // overlap borrow()/return after publication. Guarded by lock; only ever + // ticks for SF slots. private int recoveringSlots; // Resumable startup-recovery scan cursor. Advanced only by the single - // recovery driver -- the inline constructor loop (single-threaded, - // unpublished) or the PoolHousekeeper thread (the sole deferred driver) -- + // recovery driver -- the inline constructor loop followed by its private + // direct-pool thread, or the PoolHousekeeper thread (the sole deferred + // driver) -- // so the cursor itself needs no lock; the per-slot reservation it performs // (slotInUse/recoveringSlots) is still taken under `lock` because borrow() // races it. recoveryInRangeNext is the next in-range index in [0, maxSize) @@ -253,7 +281,7 @@ public SenderPool( long maxLifetimeMillis ) { this(configurationString, minSize, maxSize, acquireTimeoutMillis, - idleTimeoutMillis, maxLifetimeMillis, null, false, null, null, null); + idleTimeoutMillis, maxLifetimeMillis, null, false, null, null, null, null, null, null, null); } // Test-only constructor exposing the senderFactory seam: production builds @@ -282,8 +310,8 @@ public SenderPool( // reachable-but-not-acking server; the owner (QuestDBImpl) then drives // recovery one slot per tick on the PoolHousekeeper thread via // runStartupRecoveryStep(). White-box SF tests call this directly; the - // in-range recovery pass is concurrency-safe against borrow()/return on the - // deferred path -- see recoverOneSlotStep(). + // in-range recovery pass is concurrency-safe against borrow()/return after + // pool publication -- see recoverOneSlotStep(). @TestOnly public SenderPool( String configurationString, @@ -297,7 +325,7 @@ public SenderPool( ) { this(configurationString, minSize, maxSize, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, senderFactory, - deferStartupRecovery, null, null, null); + deferStartupRecovery, null, null, null, null, null, null, null); } // Test-only constructor adding a deterministic fault hook for the ownership @@ -316,7 +344,7 @@ public SenderPool( ) { this(configurationString, minSize, maxSize, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, senderFactory, - deferStartupRecovery, null, null, null, postFactoryHook); + deferStartupRecovery, null, null, null, postFactoryHook, null, null, null); } // Full constructor adding the user-supplied ingest callbacks (error @@ -340,7 +368,7 @@ public SenderPool( this(configurationString, minSize, maxSize, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, senderFactory, deferStartupRecovery, errorHandler, connectionListener, - drainerListener, null); + drainerListener, null, null, null, null); } private SenderPool( @@ -355,7 +383,10 @@ private SenderPool( SenderErrorHandler errorHandler, SenderConnectionListener connectionListener, BackgroundDrainerListener drainerListener, - Runnable postFactoryHook + Runnable postFactoryHook, + ThreadFactory recoveryThreadFactory, + Runnable recoveryWaiter, + Runnable beforeFailedRecoveryJoinHook ) { if (minSize < 0 || maxSize < 1 || minSize > maxSize) { throw new IllegalArgumentException("invalid pool sizing: min=" + minSize + ", max=" + maxSize); @@ -375,6 +406,10 @@ private SenderPool( this.idleTimeoutMillis = idleTimeoutMillis; this.maxLifetimeMillis = maxLifetimeMillis; this.postFactoryHook = postFactoryHook; + this.beforeFailedStartupRecoveryJoinHook = beforeFailedRecoveryJoinHook; + this.startupRecoveryWaiter = recoveryWaiter != null + ? recoveryWaiter + : this::waitForStartupRecoveryRetry; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); this.retiredSlots = new ArrayList<>(maxSize); @@ -410,17 +445,7 @@ private SenderPool( // propagate without running the cleanup below, leaking every // already-built delegate's flock + mmap'd ring + I/O thread and // resurrecting "sf slot already in use" on the next attempt. - for (int i = 0; i < built; i++) { - try { - all.get(i).delegate().close(); - } catch (Throwable ignored) { - // Best-effort cleanup: a delegate close() can throw an - // Error (e.g. an -ea AssertionError) as well as a - // RuntimeException. Swallow either so we still close the - // remaining pre-warmed slots and rethrow the original - // construction failure below. - } - } + closePrewarmedDelegates(built); throw e; } // Prewarm succeeded. Recover any unacked data a previous run left in @@ -429,18 +454,43 @@ private SenderPool( // (deferStartupRecovery=true) so QuestDB.build() never blocks on a slow // or reachable-but-not-acking server; direct constructions run it inline // here, while still single-threaded and unpublished. - if (!deferStartupRecovery) { - runStartupRecoveryToCompletion(); + Thread recoveryThread = null; + try { + if (!deferStartupRecovery) { + runStartupRecoveryToCompletion(); + if (storeAndForward && !recoveryComplete) { + ThreadFactory threadFactory = recoveryThreadFactory != null + ? recoveryThreadFactory + : SenderPool::createStartupRecoveryThread; + recoveryThread = threadFactory.newThread(this::runStartupRecoveryLoop); + recoveryThread.setDaemon(true); + } + } + this.startupRecoveryThread = recoveryThread; + if (recoveryThread != null) { + recoveryThread.start(); + } + } catch (Throwable e) { + // Thread allocation, configuration, or start can throw after + // prewarm transferred delegate ownership to this constructor. The + // pool cannot be returned, so stop a possibly-started driver before + // tearing down delegates it could otherwise still recover against. + // Preserve the construction throwable; cleanup is best-effort. + markClosing(); + stopFailedStartupRecoveryDriver(recoveryThread); + closePrewarmedDelegates(built); + throw e; } } /** - * Drives startup SF recovery to completion in a single call, bounded by one - * shared {@code acquireTimeoutMillis} wall-clock budget (and each individual - * drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used by the inline - * construction path -- single-threaded and unpublished -- and by manual / - * test drives. No-op when SF is off, the pool is shutting down, or recovery - * has already finished. Idempotent. + * Drives startup SF recovery toward completion in a single call, bounded by + * one shared {@code acquireTimeoutMillis} wall-clock budget (and each + * individual drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used by the + * inline construction path -- single-threaded and unpublished -- and by + * manual / test drives. Budget exhaustion or a transient slot failure leaves + * the cursor pending for the pool's continuing driver. No-op when SF is off, + * the pool is shutting down, or recovery has already finished. Idempotent. */ void runStartupRecoveryToCompletion() { if (!storeAndForward) { @@ -452,33 +502,64 @@ void runStartupRecoveryToCompletion() { // accepted for a single borrow, so we reuse it as the total budget; once // spent, the remaining slots wait for a later attempt (data stays // durable on disk). - final long deadlineNanos = - System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(acquireTimeoutMillis); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(acquireTimeoutMillis); + final long startNanos = System.nanoTime(); while (!closed && !recoveryComplete) { - long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); - if (remainingMillis <= 0) { + // Subtract nanoTime samples before comparing them with the duration. + // This avoids an overflowed absolute deadline when + // acquireTimeoutMillis is Long.MAX_VALUE and keeps the intended + // duration comparison explicit. + long elapsedNanos = System.nanoTime() - startNanos; + if (elapsedNanos >= budgetNanos) { LOG.warn("startup SF recovery: {}ms budget exhausted; " - + "skipping remaining slots", acquireTimeoutMillis); - recoveryComplete = true; + + "deferring remaining slots", acquireTimeoutMillis); return; } + long remainingNanos = budgetNanos - elapsedNanos; + long remainingMillis = TimeUnit.NANOSECONDS.toMillis(remainingNanos); + if (remainingMillis == 0) { + // drain() accepts milliseconds; preserve a positive sub-ms + // remainder instead of treating truncation as exhaustion. + remainingMillis = 1; + } if (!recoverOneSlotStep(Math.min(remainingMillis, RECOVERY_DRAIN_BUDGET_MILLIS))) { return; } } } + private void runStartupRecoveryLoop() { + while (!closed && !recoveryComplete) { + boolean hasImmediateWork; + try { + hasImmediateWork = runStartupRecoveryStep(); + } catch (Throwable ignored) { + // Match PoolHousekeeper: startup recovery is best-effort and a + // delegate Error must not permanently kill its only driver. + hasImmediateWork = false; + } + if (closed || recoveryComplete) { + return; + } + if (!hasImmediateWork) { + startupRecoveryWaiter.run(); + if (Thread.currentThread().isInterrupted()) { + return; + } + } + } + } + /** * Recovers at most ONE stranded managed slot and reports whether more remain. - * Driven by the {@link PoolHousekeeper} one slot per step, back-to-back, on - * the reap-loop thread: each step performs at most one drain bounded by - * {@link #RECOVERY_DRAIN_BUDGET_MILLIS} -- kept below the housekeeper - * stop/join budget -- on a delegate whose initial connect is forced OFF + * The private direct driver or {@link PoolHousekeeper} drives steps + * back-to-back: each step performs at most one drain bounded by + * {@link #RECOVERY_DRAIN_BUDGET_MILLIS} -- kept below the driver stop/join + * budget -- on a delegate whose initial connect is forced OFF * ({@link #defaultRecoverySender}) so the build makes at most one connect - * attempt. The housekeeper re-checks {@code stop} between steps while - * {@link QuestDBImpl#close()} raises the {@code closed} shutdown signal (via - * {@link #markClosing()}) BEFORE stopping it, so a {@code close()} landing - * mid-recovery normally only has to wait out a single bounded drain. The one + * attempt. Each owner raises the {@code closed} shutdown signal before + * joining its driver, so a {@code close()} landing mid-recovery normally + * only has to wait out a single bounded drain. The one * exception is an in-flight connect to a black-holed host, which blocks on * the OS connect timeout -- see the residual-window note on * {@link #recoverOneSlotStep}. @@ -519,8 +600,8 @@ boolean runStartupRecoveryStep() { *

        * The in-range pass reserves each slot index under {@code lock} for the * duration of its recovery AND counts it in the borrow() capacity check (via - * {@code recoveringSlots}), so a concurrent borrow on the deferred path can - * neither target the dir being recovered nor over-allocate past + * {@code recoveringSlots}), so a concurrent borrow after pool publication + * can neither target the dir being recovered nor over-allocate past * {@code maxSize}. Prewarmed/borrowed slots (already live, holding their * flock) are skipped, as are empty slots (a cheap directory probe); only a * slot that actually holds stranded data spends the step's single drain. The @@ -530,13 +611,14 @@ boolean runStartupRecoveryStep() { * Best-effort throughout: a build/close Error or a slow drain is logged and * never propagates, since the data stays durable on disk for a later attempt. * A build failure or drain timeout stops the current drive but leaves its - * candidate pending so a later housekeeper tick can retry after a transient + * candidate pending so a later driver attempt can retry after a transient * condition clears; it does not poison recovery for the life of the pool. *

        - * Boundedness / residual window. Recovery is driven on the - * PoolHousekeeper thread, and {@code close()} relies on a step finishing - * within {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. A step is build + - * drain + close. The drain is capped by {@link #RECOVERY_DRAIN_BUDGET_MILLIS} + * Boundedness / residual window. A continuing recovery step runs on + * either the direct pool's private driver or the PoolHousekeeper thread, and + * {@code close()} relies on a step finishing within + * {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. A step is build + drain + + * close. The drain is capped by {@link #RECOVERY_DRAIN_BUDGET_MILLIS} * and the build forces {@code initial_connect_mode=OFF} (see * {@link #defaultRecoverySender}), so it makes at most ONE connect attempt * instead of a SYNC reconnect-budget retry loop. That removes the @@ -544,7 +626,7 @@ boolean runStartupRecoveryStep() { * One residual window remains and is NOT closed here: a single in-flight * connect to a black-holed/firewalled host blocks on the OS connect timeout * (the transport exposes no application-level connect timeout to clamp it). - * If {@code close()} lands during that one connect the housekeeper join can + * If {@code close()} lands during that one connect, its driver join can * still time out and the detached build releases the slot flock shortly * after {@code close()} returns. No data is lost (the slot stays durable on * disk); the exposure is a brief "sf slot already in use" window on an @@ -567,8 +649,8 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { return false; } int i = recoveryInRangeNext; - // Reserve this index unless prewarm (or a concurrent borrow, on the - // deferred path) already holds it live. Count the reservation in + // Reserve this index unless prewarm (or a concurrent borrow after + // publication) already holds it live. Count the reservation in // recoveringSlots so the borrow() cap check cannot over-allocate // while this slot is held for recovery. boolean reserved; @@ -629,9 +711,9 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { i, maxSize - leakedSlots, maxSize, leakedSlots); } else { slotInUse[i] = false; - // On the deferred path a borrow may be waiting on capacity - // this recovery held; the freed index can now admit a - // creation. + // On a post-publication drive, a borrow may be waiting on + // capacity this recovery held; the freed index can now admit + // a creation. slotReleased.signal(); } } finally { @@ -699,6 +781,22 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { return false; } + private void closePrewarmedDelegates(int built) { + for (int i = 0; i < built; i++) { + try { + all.get(i).delegate().close(); + } catch (Throwable ignored) { + // Best-effort cleanup: one delegate close failure must not + // strand later prewarmed delegates or replace the original + // construction throwable. + } + } + } + + private static Thread createStartupRecoveryThread(Runnable runnable) { + return new Thread(runnable, "questdb-sender-pool-recovery"); + } + /** * Drains one candidate orphan slot dir within {@code remainingMillis}, * best-effort and never throwing. Builds a recoverer on {@code slotIndex} @@ -733,8 +831,8 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // Recovery delegate: forced OFF-mode initial connect (see // createRecoverer / defaultRecoverySender), so this build does // at most ONE connect attempt -- it never inherits the SYNC - // reconnect-budget retry loop that would block this - // (PoolHousekeeper) thread for minutes (M1). + // reconnect-budget retry loop that would block this recovery + // driver for minutes (M1). recoverer = createRecoverer(slotIndex); } catch (Throwable buildErr) { // A build/connect failure (e.g. server unreachable) will very @@ -921,10 +1019,10 @@ public void setBorrowWaitExpiredHook(Runnable hook) { /** * Raises the shutdown signal early -- without tearing down live delegates -- - * so an in-flight startup-recovery step driven on the {@link PoolHousekeeper} - * thread stops promptly between slots. {@link QuestDBImpl#close()} calls this - * BEFORE stopping the housekeeper, so the housekeeper join cannot time out - * waiting on a fresh slot's recovery drain. Idle-delegate teardown and the + * so an in-flight startup-recovery step stops promptly between slots. Direct + * {@link #close()} calls this before joining its private driver; + * {@link QuestDBImpl#close()} calls it before stopping the housekeeper. + * Idle-delegate teardown and the * outstanding-lease wait still happen in {@link #close()} (guarded by * {@code closeStarted}, so this early signal never short-circuits them); * borrowed delegates returned after this signal are torn down on the @@ -933,6 +1031,9 @@ public void setBorrowWaitExpiredHook(Runnable hook) { */ void markClosing() { closed = true; + synchronized (startupRecoverySignal) { + startupRecoverySignal.notifyAll(); + } } /** @@ -959,6 +1060,11 @@ void markClosing() { */ @Override public void close() { + // Direct pools own their continuing recovery driver. Stop it before + // snapshotting/closing delegates; deferred pools have a null thread and + // QuestDBImpl stops their external PoolHousekeeper before calling here. + markClosing(); + stopStartupRecoveryDriver(); SenderSlot[] idleSnapshot; lock.lock(); try { @@ -1099,6 +1205,62 @@ public void giveBack(PooledSender ps) { retireLease(ps, " during pool shutdown"); } + @TestOnly + private void setStartupRecoveryJoinHooks(Runnable beforeJoinHook, Runnable afterJoinHook) { + this.beforeStartupRecoveryJoinHook = beforeJoinHook; + this.afterStartupRecoveryJoinHook = afterJoinHook; + } + + private void stopFailedStartupRecoveryDriver(Thread recoveryThread) { + if (recoveryThread == null || recoveryThread == Thread.currentThread()) { + return; + } + if (beforeFailedStartupRecoveryJoinHook != null) { + beforeFailedStartupRecoveryJoinHook.run(); + } + boolean interrupted = false; + while (recoveryThread.isAlive()) { + try { + recoveryThread.join(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private void stopStartupRecoveryDriver() { + if (startupRecoveryThread == null || startupRecoveryThread == Thread.currentThread()) { + return; + } + if (beforeStartupRecoveryJoinHook != null) { + beforeStartupRecoveryJoinHook.run(); + } + try { + startupRecoveryThread.join(PoolHousekeeper.STOP_TIMEOUT_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (afterStartupRecoveryJoinHook != null) { + afterStartupRecoveryJoinHook.run(); + } + } + + private void waitForStartupRecoveryRetry() { + synchronized (startupRecoverySignal) { + if (closed || recoveryComplete) { + return; + } + try { + startupRecoverySignal.wait(RECOVERY_RETRY_INTERVAL_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + /** * Retires one lease on the calling (borrower's) thread: validates the * lease generation under the lock, removes the slot from {@code all}, @@ -1361,9 +1523,10 @@ private Sender defaultSender(int slotIndex) { /** * Same managed-slot delegate as {@link #defaultSender}, but with the * initial connect forced to {@link Sender.InitialConnectMode#OFF}. Used - * only for startup recovery, which runs on the PoolHousekeeper thread: OFF - * makes the build do at most ONE connect attempt instead of retrying for - * the whole reconnect budget (SYNC, auto-enabled by any reconnect_* knob), + * only for startup recovery, which runs on a private direct driver or the + * PoolHousekeeper thread: OFF makes the build do at most ONE connect attempt + * instead of retrying for the whole reconnect budget (SYNC, auto-enabled by + * any reconnect_* knob), * keeping a recovery step bounded below * {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. See M1 / the residual-window * note on {@link #recoverOneSlotStep}. @@ -1432,8 +1595,8 @@ private Sender buildManagedSlotSender(int slotIndex, boolean forRecovery) { // its OWN slot (the one recoverOneSlotStep is processing); it must // never start a BackgroundDrainerPool for sibling/foreign orphans. // If it did, the delegate's close() -- called from - // drainCandidateSlotForRecovery() on the PoolHousekeeper thread, - // BEFORE its cursorEngine.close() releases the slot flock -- would + // drainCandidateSlotForRecovery() on the recovery driver, BEFORE + // its cursorEngine.close() releases the slot flock -- would // block in BackgroundDrainerPool.close() for up to // GRACEFUL_DRAIN_MILLIS + STOP_GRACE_MILLIS (3s) against a // reachable-but-not-acking server. That overruns a recovery step's diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index ea7e901a..78eb44a5 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -47,7 +47,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.nio.ByteBuffer; @@ -58,6 +60,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -2633,6 +2636,355 @@ public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws E }); } + @Test + public void testDirectRecoveryContinuesAfterFiniteBudgetExhaustion() throws Exception { + createCandidateSlot("default-0"); + createCandidateSlot("default-1"); + CountDownLatch drained = new CountDownLatch(2); + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + IntFunction factory = idx -> { + attempts[idx].incrementAndGet(); + return successfulRecoverySender(idx, drained); + }; + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + + try (SenderPool pool = newPoolWithFactory(config, 0, 1, 0, factory)) { + Assert.assertTrue("direct pool must continue recovery after its inline budget expires", + drained.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("in-range candidate must be recovered", 1, attempts[0].get()); + Assert.assertEquals("out-of-range candidate must be recovered", 1, attempts[1].get()); + } + } + + @Test + public void testDirectRecoveryCloseJoinsDriverAndStopsRemainingCandidates() throws Exception { + createCandidateSlot("default-0"); + createCandidateSlot("default-1"); + CountDownLatch afterJoin = new CountDownLatch(1); + CountDownLatch beforeJoin = new CountDownLatch(1); + CountDownLatch closeReturned = new CountDownLatch(1); + CountDownLatch delegateCloseStarted = new CountDownLatch(1); + CountDownLatch drainStarted = new CountDownLatch(1); + CountDownLatch releaseDelegateClose = new CountDownLatch(1); + CountDownLatch releaseDrain = new CountDownLatch(1); + AtomicBoolean driverAliveAfterJoin = new AtomicBoolean(); + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + AtomicReference closeFailure = new AtomicReference<>(); + IntFunction senderFactory = idx -> { + attempts[idx].incrementAndGet(); + return blockingFakeSender( + idx, drainStarted, releaseDrain, delegateCloseStarted, releaseDelegateClose); + }; + + SenderPool pool = newPoolWithFactory( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 0, 2, 0, senderFactory); + Thread recoveryThread = (Thread) getField(pool, "startupRecoveryThread"); + invokeSetStartupRecoveryJoinHooks( + pool, beforeJoin::countDown, + () -> { + driverAliveAfterJoin.set(recoveryThread.isAlive()); + afterJoin.countDown(); + }); + Thread closeThread = new Thread(() -> { + try { + pool.close(); + } catch (Throwable t) { + closeFailure.set(t); + } finally { + closeReturned.countDown(); + } + }, "test-direct-pool-close"); + try { + Assert.assertTrue("direct recovery driver must enter the first drain", + drainStarted.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("direct recovery driver must be running", recoveryThread.isAlive()); + + closeThread.start(); + Assert.assertTrue("close must enter its direct-driver join operation", + beforeJoin.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("close itself must raise the shutdown signal before joining", + (Boolean) getField(pool, "closed")); + + releaseDrain.countDown(); + Assert.assertTrue("driver must remain deliberately held before termination", + delegateCloseStarted.await(10, TimeUnit.SECONDS)); + Assert.assertEquals("close must not return while its driver is held", + 1L, closeReturned.getCount()); + + releaseDelegateClose.countDown(); + Assert.assertTrue("close must return after the driver is released", + closeReturned.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("close must complete its join operation", + afterJoin.await(10, TimeUnit.SECONDS)); + if (closeFailure.get() != null) { + throw new AssertionError("close failed", closeFailure.get()); + } + Assert.assertFalse("the recovery driver must be dead when close's join returns", + driverAliveAfterJoin.get()); + Assert.assertFalse("close must leave the direct recovery driver quiescent", + recoveryThread.isAlive()); + Assert.assertEquals("the in-flight candidate must have been built", 1, attempts[0].get()); + Assert.assertEquals("shutdown must prevent a later candidate build", 0, attempts[1].get()); + } finally { + releaseDrain.countDown(); + releaseDelegateClose.countDown(); + closeThread.join(TimeUnit.SECONDS.toMillis(10)); + pool.close(); + } + } + + @Test + public void testDirectRecoveryFailedAttemptEntersRetryWait() throws Throwable { + createCandidateSlot("default-0"); + CountDownLatch beforeJoin = new CountDownLatch(1); + CountDownLatch closeReturned = new CountDownLatch(1); + CountDownLatch recoveryOperationObserved = new CountDownLatch(1); + CountDownLatch releaseWait = new CountDownLatch(1); + CountDownLatch waitEntered = new CountDownLatch(1); + AtomicInteger attempts = new AtomicInteger(); + AtomicReference closeFailure = new AtomicReference<>(); + IntFunction senderFactory = idx -> { + if (attempts.incrementAndGet() > 1) { + recoveryOperationObserved.countDown(); + } + throw new LineSenderException("injected recovery failure"); + }; + Runnable recoveryWaiter = () -> { + waitEntered.countDown(); + recoveryOperationObserved.countDown(); + try { + releaseWait.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }; + + SenderPool pool = newPoolWithRecoveryControls( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 0, 1, 0, senderFactory, null, recoveryWaiter, null); + invokeSetStartupRecoveryJoinHooks(pool, beforeJoin::countDown, null); + Thread closeThread = new Thread(() -> { + try { + pool.close(); + } catch (Throwable t) { + closeFailure.set(t); + } finally { + closeReturned.countDown(); + } + }, "test-retry-wait-close"); + try { + Assert.assertTrue("recovery must either wait or retry after the failed attempt", + recoveryOperationObserved.await(10, TimeUnit.SECONDS)); + Assert.assertEquals("a failed recovery attempt must enter the retry-wait operation", + 0L, waitEntered.getCount()); + Assert.assertEquals("the driver must wait immediately after its first failed attempt", + 1, attempts.get()); + + closeThread.start(); + Assert.assertTrue("close must reach the driver join while the waiter is held", + beforeJoin.await(10, TimeUnit.SECONDS)); + releaseWait.countDown(); + Assert.assertTrue("close must finish after the retry waiter is released", + closeReturned.await(10, TimeUnit.SECONDS)); + if (closeFailure.get() != null) { + throw new AssertionError("close failed", closeFailure.get()); + } + Assert.assertEquals("shutdown must prevent another recovery attempt", 1, attempts.get()); + } finally { + releaseWait.countDown(); + closeThread.join(TimeUnit.SECONDS.toMillis(10)); + pool.close(); + } + } + + @Test + public void testDirectRecoveryThreadCreationFailureClosesPrewarmedDelegates() throws Throwable { + createCandidateSlot("default-2"); + AssertionError failure = new AssertionError("injected recovery thread creation failure"); + AtomicInteger[] closeCalls = {new AtomicInteger(), new AtomicInteger()}; + IntFunction senderFactory = idx -> closeCountingSender(idx, closeCalls, idx == 0); + ThreadFactory threadFactory = runnable -> { + throw failure; + }; + + try { + newPoolWithRecoveryThreadFactory( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 2, 2, 0, senderFactory, threadFactory); + Assert.fail("construction must propagate the recovery thread creation failure"); + } catch (AssertionError actual) { + Assert.assertSame("construction must preserve throwable identity", failure, actual); + } + Assert.assertEquals("cleanup must attempt the first prewarmed delegate", 1, closeCalls[0].get()); + Assert.assertEquals("one close failure must not strand the next delegate", 1, closeCalls[1].get()); + } + + @Test + public void testDirectRecoveryThreadCreationFailureReleasesPrewarmedSfFlocks() throws Exception { + TestUtils.assertMemoryLeak(() -> { + createCandidateSlot("default-2"); + AssertionError failure = new AssertionError("injected recovery thread creation failure"); + ThreadFactory threadFactory = runnable -> { + throw failure; + }; + + try (TestWebSocketServer ack = new TestWebSocketServer(new CountingAckHandler())) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + try { + newPoolWithRecoveryThreadFactory(config, 2, 2, 0, null, threadFactory); + Assert.fail("construction must propagate the recovery thread creation failure"); + } catch (AssertionError actual) { + Assert.assertSame("construction must preserve throwable identity", failure, actual); + } catch (Throwable unexpected) { + throw new AssertionError("unexpected construction failure", unexpected); + } + + try (SlotLock ignored0 = SlotLock.acquire(slot("default-0")); + SlotLock ignored1 = SlotLock.acquire(slot("default-1"))) { + // Reacquisition proves failed construction closed both real + // prewarmed delegates and released their SF flocks. + } + } + }); + } + + @Test + public void testDirectRecoveryThreadStartFailureClosesPrewarmedDelegates() throws Throwable { + createCandidateSlot("default-2"); + AssertionError failure = new AssertionError("injected recovery thread start failure"); + AtomicBoolean cleanupBeforeDriverQuiescence = new AtomicBoolean(); + AtomicInteger failedJoinCalls = new AtomicInteger(); + AtomicInteger[] closeCalls = {new AtomicInteger(), new AtomicInteger()}; + AtomicReference recoveryThread = new AtomicReference<>(); + CountDownLatch releaseDriver = new CountDownLatch(1); + CountDownLatch running = new CountDownLatch(1); + IntFunction senderFactory = idx -> closeCountingSender( + idx, closeCalls, false, recoveryThread, cleanupBeforeDriverQuiescence, releaseDriver); + ThreadFactory threadFactory = runnable -> { + Thread thread = new Thread(() -> { + running.countDown(); + try { + releaseDriver.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + runnable.run(); + }, "test-started-recovery-driver") { + @Override + public synchronized void start() { + super.start(); + try { + if (!running.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("recovery driver did not start"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("interrupted waiting for recovery driver", e); + } + throw failure; + } + }; + recoveryThread.set(thread); + return thread; + }; + Runnable beforeFailedJoin = () -> { + failedJoinCalls.incrementAndGet(); + releaseDriver.countDown(); + }; + + try { + newPoolWithRecoveryControls( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 2, 2, 0, senderFactory, threadFactory, null, beforeFailedJoin); + Assert.fail("construction must propagate the recovery thread start failure"); + } catch (AssertionError actual) { + Assert.assertSame("construction must preserve throwable identity", failure, actual); + } finally { + releaseDriver.countDown(); + } + Assert.assertEquals("constructor cleanup must enter the failed-driver join", 1, failedJoinCalls.get()); + Assert.assertFalse("delegate cleanup must not begin while the failed driver is alive", + cleanupBeforeDriverQuiescence.get()); + Assert.assertFalse("a possibly-started failed driver must terminate before delegate cleanup", + recoveryThread.get().isAlive()); + Assert.assertEquals("cleanup must close the first prewarmed delegate", 1, closeCalls[0].get()); + Assert.assertEquals("cleanup must close the second prewarmed delegate", 1, closeCalls[1].get()); + } + + @Test + public void testDirectRecoveryRetriesTransientFailureAndRemainingSlots() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + for (int i = 0; i < 2; i++) { + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-" + i + ";close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", i).atNow(); + seed.flush(); + } + } + } + Assert.assertTrue("in-range fixture must contain unacked data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + CountDownLatch drained = new CountDownLatch(2); + IntFunction factory = idx -> { + int attempt = attempts[idx].incrementAndGet(); + if (idx == 0 && attempt == 1) { + throw new LineSenderException("transient direct recovery failure"); + } + Sender delegate = Sender.builder(config).senderId("default-" + idx).build(); + return notifyingCloseSender(delegate, drained); + }; + + try (SenderPool pool = newPoolWithFactory(config, 0, 1, 5_000, factory)) { + Assert.assertTrue("both recovery delegates must drain and close", + drained.await(15, TimeUnit.SECONDS)); + Assert.assertFalse("failed in-range candidate must be retried and delivered", + hasSegmentFile(slot("default-0"))); + Assert.assertFalse("remaining out-of-range candidate must also be delivered", + hasSegmentFile(slot("default-1"))); + Assert.assertEquals("failed in-range candidate must be retried", 2, attempts[0].get()); + Assert.assertEquals("remaining out-of-range candidate must also be recovered", + 1, attempts[1].get()); + Assert.assertTrue("both recovered frames must reach the server", + handler.frames.get() >= 2); + } + } + }); + } + + @Test + public void testLongMaxStartupRecoveryBudgetDoesNotOverflow() throws Exception { + createCandidateSlot("default-0"); + AtomicInteger attempts = new AtomicInteger(); + IntFunction factory = idx -> { + attempts.incrementAndGet(); + return successfulRecoverySender(idx, new CountDownLatch(0)); + }; + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + + try (SenderPool pool = newPoolWithFactory(config, 0, 1, Long.MAX_VALUE, factory)) { + Assert.assertEquals("Long.MAX_VALUE must leave a positive inline recovery budget", + 1, attempts.get()); + Assert.assertTrue("the inline scan must complete", (Boolean) getField(pool, "recoveryComplete")); + } + } + @Test public void testStartupRecoveryIsBoundedByASharedBudget() throws Exception { // Regression for the startup-recovery budget (M1). @@ -3048,10 +3400,97 @@ public void testCloseDuringDeferredRecoveryStopsBuildingOnClosingPool() throws E // Helpers. // ---------------------------------------------------------------------- + private static Sender closeCountingSender( + int idx, AtomicInteger[] closeCalls, boolean throwOnClose + ) { + return closeCountingSender(idx, closeCalls, throwOnClose, null, null, null); + } + + private static Sender closeCountingSender( + int idx, + AtomicInteger[] closeCalls, + boolean throwOnClose, + AtomicReference recoveryThread, + AtomicBoolean cleanupBeforeDriverQuiescence, + CountDownLatch releaseDriver + ) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + if (recoveryThread != null && recoveryThread.get().isAlive()) { + cleanupBeforeDriverQuiescence.set(true); + releaseDriver.countDown(); + } + closeCalls[idx].incrementAndGet(); + if (throwOnClose) { + throw new AssertionError("injected delegate close failure"); + } + return null; + case "toString": + return "CloseCountingSender-" + idx; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + throw new AssertionError("unexpected prewarmed sender call: " + method.getName()); + } + }); + } + + private void createCandidateSlot(String name) throws IOException { + java.nio.file.Path dir = Paths.get(slot(name)); + java.nio.file.Files.createDirectories(dir); + java.nio.file.Files.write(dir.resolve("0.sfa"), new byte[]{1}); + } + + private static Sender notifyingCloseSender(Sender delegate, CountDownLatch closed) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + try { + Object result = method.invoke(delegate, args); + if ("close".equals(method.getName())) { + closed.countDown(); + } + return result; + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + } + private String slot(String name) { return sfDir + "/" + name; } + private static Sender successfulRecoverySender(int idx, CountDownLatch drained) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "drain": + drained.countDown(); + return true; + case "close": + return null; + case "toString": + return "SuccessfulRecoverySender-" + idx; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + throw new AssertionError("unexpected recovery sender call: " + method.getName()); + } + }); + } + private void assertPreallocatedExitHandoffCleansStartupRecoverer( int strandedIndex, int maxSize) throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -3362,6 +3801,46 @@ private static SenderPool newPoolWithFactory( return new SenderPool(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, senderFactory); } + private static SenderPool newPoolWithRecoveryControls( + String cfg, + int min, + int max, + long acquireMs, + IntFunction senderFactory, + ThreadFactory threadFactory, + Runnable recoveryWaiter, + Runnable beforeFailedRecoveryJoinHook + ) throws Throwable { + Constructor constructor = SenderPool.class.getDeclaredConstructor( + String.class, int.class, int.class, long.class, long.class, long.class, + IntFunction.class, boolean.class, + io.questdb.client.SenderErrorHandler.class, + io.questdb.client.SenderConnectionListener.class, + io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerListener.class, + Runnable.class, ThreadFactory.class, Runnable.class, Runnable.class); + constructor.setAccessible(true); + try { + return constructor.newInstance( + cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, + senderFactory, false, null, null, null, null, threadFactory, + recoveryWaiter, beforeFailedRecoveryJoinHook); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + + private static SenderPool newPoolWithRecoveryThreadFactory( + String cfg, + int min, + int max, + long acquireMs, + IntFunction senderFactory, + ThreadFactory threadFactory + ) throws Throwable { + return newPoolWithRecoveryControls( + cfg, min, max, acquireMs, senderFactory, threadFactory, null, null); + } + // Uses the @TestOnly 8-arg constructor (deferStartupRecovery=true) so a test // can build a pool whose SF startup recovery is NOT run inline -- mirroring // the pooled QuestDB handle, which defers it to the housekeeper. @@ -3394,6 +3873,15 @@ private static void invokeMarkClosing(SenderPool pool) throws Exception { m.invoke(pool); } + private static void invokeSetStartupRecoveryJoinHooks( + SenderPool pool, Runnable beforeJoinHook, Runnable afterJoinHook + ) throws Exception { + Method m = SenderPool.class.getDeclaredMethod( + "setStartupRecoveryJoinHooks", Runnable.class, Runnable.class); + m.setAccessible(true); + m.invoke(pool, beforeJoinHook, afterJoinHook); + } + // Deferred pool (deferStartupRecovery=true) WITH an injected factory, so a // test can drive the housekeeper recovery path against fully controlled // (fake) recoverers. @@ -3402,9 +3890,23 @@ private static SenderPool newDeferredPoolWithFactory( return new SenderPool(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, factory, true); } - // Fake Sender whose drain() (for slot 0 only) parks until released, opening a - // deterministic shutdown-during-recovery window. Holds no native resources. - private static Sender blockingFakeSender(int idx, CountDownLatch drainStarted, CountDownLatch release) { + private static Sender blockingFakeSender( + int idx, CountDownLatch drainStarted, CountDownLatch releaseDrain + ) { + return blockingFakeSender( + idx, drainStarted, releaseDrain, new CountDownLatch(0), new CountDownLatch(0)); + } + + // Fake Sender whose drain() and close() (for slot 0 only) park until + // released, opening deterministic shutdown and pre-termination windows. + // Holds no native resources. + private static Sender blockingFakeSender( + int idx, + CountDownLatch drainStarted, + CountDownLatch releaseDrain, + CountDownLatch closeStarted, + CountDownLatch releaseClose + ) { return (Sender) java.lang.reflect.Proxy.newProxyInstance( Sender.class.getClassLoader(), new Class[]{Sender.class}, @@ -3413,10 +3915,14 @@ private static Sender blockingFakeSender(int idx, CountDownLatch drainStarted, C case "drain": if (idx == 0) { drainStarted.countDown(); - release.await(); + releaseDrain.await(); } return true; case "close": + if (idx == 0) { + closeStarted.countDown(); + releaseClose.await(); + } return null; case "toString": return "BlockingFakeSender-" + idx;