From 70dfdf496530a445da3772ac26fabbdf232eda06 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:44:00 +0800 Subject: [PATCH 01/40] feat(persistence): add optional bounded journal paging --- docs/state-store-sqlite.md | 14 +++ eng/api-baseline.json | 8 ++ .../SqliteCloudStateTransaction.cs | 52 ++++++++- .../CloudOperationJournalPaging.cs | 103 +++++++++++++++++ .../SqliteCloudStateStoreTests.cs | 23 ++++ .../CloudOperationJournalPageTests.cs | 23 ++++ .../CloudStateStoreContractTests.cs | 104 ++++++++++++++++++ .../InMemoryCloudStateStoreContractTests.cs | 28 +++++ 8 files changed, 354 insertions(+), 1 deletion(-) create mode 100644 src/CfSharp/Persistence/CloudOperationJournalPaging.cs create mode 100644 tests/CfSharp.Tests/Persistence/CloudOperationJournalPageTests.cs diff --git a/docs/state-store-sqlite.md b/docs/state-store-sqlite.md index 66afd4a..2566842 100644 --- a/docs/state-store-sqlite.md +++ b/docs/state-store-sqlite.md @@ -35,6 +35,20 @@ Applications that need another database can implement `ICloudStateStoreFactory`, `ICloudStateStore`, and `ICloudStateTransaction`. Preserve the same commit, rollback, ownership, checkpoint, and retry semantics; the high-level API does not require SQLite. +### Optional bounded journal paging + +SQLite implements `ICloudOperationJournalPaging` on each transaction's `Operations` repository. +Capture `GetHighWaterSequenceAsync()` once, then call `ReadPageAsync(after, through, limit)` with +an exclusive lower bound, the captured inclusive upper bound, and a limit from 1 through 4096. +Use `LastScannedSequence` directly as the next lower bound. The sequence index bounds both the +query and its memory use; one extra row determines `HasMore` without advancing past that row. +Later enqueues stay outside the scan, and concurrent acknowledgements may leave sequence gaps or +an empty next page. A terminal empty page returns the upper bound and never waits. + +Pages own no reader or transaction and do not acknowledge operations or update item revisions. +The legacy `ICloudOperationJournal` contract is unchanged: custom stores may opt into paging on +their repository, preserving monotonic enqueue sequences and transactional read semantics. + ### Schema 5 recovery fence Schema 5 preserves the existing table layout and data while fencing the remote placeholder creation and observation-reconciliation protocol. Versions 0 through 4 upgrade in place. Older libraries reject the newer version rather than ignore pending creation records. Do not downgrade the schema number manually. Back up state before upgrading and use matching core and SQLite packages. diff --git a/eng/api-baseline.json b/eng/api-baseline.json index 74f28ff..9b30f69 100644 --- a/eng/api-baseline.json +++ b/eng/api-baseline.json @@ -1111,6 +1111,11 @@ "property System.ReadOnlyMemory\u003CSystem.Byte\u003E Payload()", "property System.Nullable\u003CSystem.DateTimeOffset\u003E RetryAfter()", "property System.Int64 Sequence()", + "type CfSharp.CloudOperationJournalPage", + "ctor CfSharp.CloudOperationJournalPage(System.Collections.Generic.IReadOnlyList\u003CCfSharp.CloudOperationJournalEntry\u003E operations,System.Int64 lastScannedSequence,System.Boolean hasMore)", + "property System.Boolean HasMore()", + "property System.Int64 LastScannedSequence()", + "property System.Collections.Generic.IReadOnlyList\u003CCfSharp.CloudOperationJournalEntry\u003E Operations()", "type CfSharp.CloudPinState", "field CfSharp.CloudPinState Excluded", "field CfSharp.CloudPinState Pinned", @@ -1772,6 +1777,9 @@ "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudOperationJournalEntry\u003E GetAsync(System.Guid operationId,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask\u003CSystem.Collections.Generic.IReadOnlyList\u003CCfSharp.CloudOperationJournalEntry\u003E\u003E ListAsync(System.Int32 maximumCount,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask\u003CSystem.Collections.Generic.IReadOnlyList\u003CCfSharp.CloudOperationJournalEntry\u003E\u003E ListByItemIdAsync(System.Guid itemId,System.Int32 maximumCount,System.Threading.CancellationToken cancellationToken)", + "type CfSharp.ICloudOperationJournalPaging", + "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudOperationJournalPage\u003E ReadPageAsync(System.Int64 afterSequence,System.Int64 throughSequence,System.Int32 limit,System.Threading.CancellationToken cancellationToken)", + "method System.Threading.Tasks.ValueTask\u003CSystem.Int64\u003E GetHighWaterSequenceAsync(System.Threading.CancellationToken cancellationToken)", "type CfSharp.ICloudRemoteBatchRepository", "method System.Threading.Tasks.ValueTask RemoveAsync(System.String batchId,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask UpsertAsync(CfSharp.CloudRemoteBatchState batch,System.Threading.CancellationToken cancellationToken)", diff --git a/src/CfSharp.Storage.Sqlite/SqliteCloudStateTransaction.cs b/src/CfSharp.Storage.Sqlite/SqliteCloudStateTransaction.cs index c6fd539..2865edf 100644 --- a/src/CfSharp.Storage.Sqlite/SqliteCloudStateTransaction.cs +++ b/src/CfSharp.Storage.Sqlite/SqliteCloudStateTransaction.cs @@ -532,7 +532,7 @@ await ExecuteDeleteAsync( } } - private sealed class OperationRepository : ICloudOperationJournal + private sealed class OperationRepository : ICloudOperationJournal, ICloudOperationJournalPaging { private const string SelectColumns = """ SELECT operation_id, kind, item_id, payload, created_at_ticks, @@ -636,6 +636,56 @@ public async ValueTask> ListAsync( cancellationToken).ConfigureAwait(false); } + public async ValueTask GetHighWaterSequenceAsync(CancellationToken cancellationToken = default) => + await _owner.ExecuteAsync( + async token => + { + await using SqliteCommand command = _owner.CreateCommand( + "SELECT COALESCE(MAX(sequence), 0) FROM operations;", token); + return Convert.ToInt64(await command.ExecuteScalarAsync(token).ConfigureAwait(false), CultureInfo.InvariantCulture); + }, + "The SQLite journal boundary could not be read.", + cancellationToken).ConfigureAwait(false); + + public async ValueTask ReadPageAsync( + long afterSequence, + long throughSequence, + int limit, + CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegative(afterSequence); + ArgumentOutOfRangeException.ThrowIfLessThan(throughSequence, afterSequence); + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(limit, 4096); + return await _owner.ExecuteAsync( + async token => + { + // The extra row is only a continuation probe. Do not deliver it or advance + // the cursor past it: callers must be able to reach it after an ACK hole. + await using SqliteCommand command = _owner.CreateCommand( + SelectColumns + " WHERE sequence > $after AND sequence <= $through ORDER BY sequence LIMIT $count;", token); + command.Parameters.AddWithValue("$after", afterSequence); + command.Parameters.AddWithValue("$through", throughSequence); + command.Parameters.AddWithValue("$count", limit + 1); + await using SqliteDataReader reader = await command.ExecuteReaderAsync(token).ConfigureAwait(false); + List rows = []; + while (await reader.ReadAsync(token).ConfigureAwait(false)) + { + rows.Add(ReadOperation(reader)); + } + + bool hasMore = rows.Count > limit; + if (hasMore) + { + rows.RemoveAt(rows.Count - 1); + } + + return new CloudOperationJournalPage(rows, rows.Count == 0 ? throughSequence : rows[^1].Sequence, hasMore); + }, + "The SQLite journal page could not be read.", + cancellationToken).ConfigureAwait(false); + } + public async ValueTask> ListByItemIdAsync( Guid itemId, int maximumCount, diff --git a/src/CfSharp/Persistence/CloudOperationJournalPaging.cs b/src/CfSharp/Persistence/CloudOperationJournalPaging.cs new file mode 100644 index 0000000..16162a7 --- /dev/null +++ b/src/CfSharp/Persistence/CloudOperationJournalPaging.cs @@ -0,0 +1,103 @@ +using System.Collections.ObjectModel; + +namespace CfSharp; + +/// Provides optional, bounded keyset reads of a transactional operation journal. +/// +/// Implement this capability on to support local +/// change scans without changing the legacy journal contract. Calls and returned repositories +/// have the owning transaction's single-consumer lifetime. Reads never acknowledge operations, +/// change their sequence or payload, or advance item revisions. Custom implementations must query +/// a bounded range rather than load the complete journal. No Windows API is invoked by this contract. +/// +public interface ICloudOperationJournalPaging +{ + /// Reads the largest sequence currently present in the transaction's journal. + /// Token that cancels the read without changing the journal. + /// The largest pending sequence, or zero for an empty journal. + /// + /// Capture this value once per scan. Monotonically assigned enqueue sequences keep later writes + /// outside that scan even when acknowledged rows disappear. This is a delivery boundary, not a + /// snapshot of item associations or permission to execute an operation. + /// + /// The owning transaction has terminated. + /// The read was canceled. + ValueTask GetHighWaterSequenceAsync(CancellationToken cancellationToken = default); + + /// Reads an ascending page with an exclusive lower and inclusive upper bound. + /// Nonnegative exclusive lower bound; use zero for the first page. + /// Nonnegative inclusive scan boundary, at least the lower bound. + /// Maximum delivered row count, from one through 4096. + /// Token that cancels the query and releases its reader. + /// An owned, handle-free page. Empty terminal pages never wait for another enqueue. + /// + /// All operation kinds remain in sequence order. An implementation may read one additional row + /// to determine continuation, but must not advance the cursor past an undelivered row. HasMore + /// describes this transaction: concurrent acknowledgements may make the next page empty. + /// + /// The bounds or limit are invalid. + /// The owning transaction has terminated. + /// The query was canceled. + ValueTask ReadPageAsync( + long afterSequence, + long throughSequence, + int limit, + CancellationToken cancellationToken = default); +} + +/// Contains one immutable, bounded transactional journal delivery. +/// +/// This value copies its row collection, owns no storage resources, and is safe for concurrent +/// reads after its originating transaction ends. ItemId retains the journal's nullable association +/// semantics; it is not native object proof. Storage implementations construct this value after +/// enforcing the requested scan bounds and limit. +/// +public sealed class CloudOperationJournalPage +{ + /// Copies a validated ascending delivery and its continuation facts. + /// Delivered rows with positive, strictly ascending sequences. + /// + /// The last delivered sequence, or the scan's inclusive upper bound for a terminal empty page. + /// + /// Whether an undelivered row remains inside the bounded range. + /// The collection is null. + /// Rows, ordering, or continuation facts are inconsistent. + /// The cursor is negative. + public CloudOperationJournalPage( + IReadOnlyList operations, + long lastScannedSequence, + bool hasMore) + { + ArgumentNullException.ThrowIfNull(operations); + ArgumentOutOfRangeException.ThrowIfNegative(lastScannedSequence); + CloudOperationJournalEntry[] rows = operations.ToArray(); + long previous = 0; + foreach (CloudOperationJournalEntry row in rows) + { + if (row is null || row.Sequence <= previous) + { + throw new ArgumentException("Journal page sequences must be positive and strictly ascending.", nameof(operations)); + } + + previous = row.Sequence; + } + + if ((rows.Length != 0 && lastScannedSequence != previous) || (hasMore && rows.Length == 0)) + { + throw new ArgumentException("Journal page continuation must follow the last delivered row.", nameof(lastScannedSequence)); + } + + Operations = new ReadOnlyCollection(rows); + LastScannedSequence = lastScannedSequence; + HasMore = hasMore; + } + + /// Gets the owned, strictly ordered delivered operations. + public IReadOnlyList Operations { get; } + + /// Gets the exclusive lower bound to use for the next page, without adding one. + public long LastScannedSequence { get; } + + /// Gets whether another row remained in the range at the page's read point. + public bool HasMore { get; } +} diff --git a/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs index 0f859b0..211a482 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs @@ -54,6 +54,29 @@ protected override ICloudStateStoreFactory CreateFactory() => protected override CloudStateStoreContext CreateContext() => new(_syncRootPath); + [Fact] + public async Task JournalPagingUsesSequenceIndexWithoutTemporarySort() + { + await using (ICloudStateStore store = await CreateFactory().OpenAsync(CreateContext())) + { + } + + await using SqliteConnection connection = new($"Data Source={_databasePath};Pooling=False"); + await connection.OpenAsync(); + await using SqliteCommand command = connection.CreateCommand(); + command.CommandText = "EXPLAIN QUERY PLAN SELECT sequence, operation_id FROM operations " + + "WHERE sequence > 1 AND sequence <= 100000 ORDER BY sequence LIMIT 8;"; + await using SqliteDataReader reader = await command.ExecuteReaderAsync(); + List plan = []; + while (await reader.ReadAsync()) + { + plan.Add(reader.GetString(3)); + } + + Assert.Contains(plan, detail => detail.Contains("SEARCH operations USING INTEGER PRIMARY KEY", StringComparison.Ordinal)); + Assert.DoesNotContain(plan, detail => detail.Contains("TEMP B-TREE", StringComparison.Ordinal)); + } + [Fact] public async Task CommitDoesNotReportCleanupCallbackFailureAsCommitFailure() { diff --git a/tests/CfSharp.Tests/Persistence/CloudOperationJournalPageTests.cs b/tests/CfSharp.Tests/Persistence/CloudOperationJournalPageTests.cs new file mode 100644 index 0000000..720630d --- /dev/null +++ b/tests/CfSharp.Tests/Persistence/CloudOperationJournalPageTests.cs @@ -0,0 +1,23 @@ +namespace CfSharp.Tests.Persistence; + +public sealed class CloudOperationJournalPageTests +{ + [Fact] + public void PageOwnsCollectionAndRejectsSkippedContinuation() + { + CloudOperationJournalEntry first = CreateEntry(1); + CloudOperationJournalEntry second = CreateEntry(3); + CloudOperationJournalEntry[] input = [first, second]; + CloudOperationJournalPage page = new(input, 3, true); + input[0] = second; + Assert.Same(first, page.Operations[0]); + Assert.Throws(() => new CloudOperationJournalPage([second, first], 1, false)); + Assert.Throws(() => new CloudOperationJournalPage([first, first], 1, false)); + Assert.Throws(() => new CloudOperationJournalPage([first], 2, true)); + Assert.Throws(() => new CloudOperationJournalPage([], 3, true)); + Assert.Throws(() => new CloudOperationJournalPage([], -1, false)); + } + + private static CloudOperationJournalEntry CreateEntry(long sequence) => + new(Guid.NewGuid(), CloudStateOperationKind.Create, null, [1], DateTimeOffset.UtcNow, sequence: sequence); +} diff --git a/tests/CfSharp.Tests/Persistence/CloudStateStoreContractTests.cs b/tests/CfSharp.Tests/Persistence/CloudStateStoreContractTests.cs index 153ba15..f7b685d 100644 --- a/tests/CfSharp.Tests/Persistence/CloudStateStoreContractTests.cs +++ b/tests/CfSharp.Tests/Persistence/CloudStateStoreContractTests.cs @@ -4,6 +4,110 @@ public abstract class CloudStateStoreContractTests { protected abstract ICloudStateStoreFactory CreateFactory(); + [Fact] + public async Task JournalPagingTraversesBacklogWithoutAcknowledgingOrIncludingLaterWrites() + { + await using ICloudStateStore store = await CreateFactory().OpenAsync(CreateContext()); + List expected = []; + CloudStateOperationKind[] kinds = Enum.GetValues(); + await using (ICloudStateTransaction write = await store.BeginTransactionAsync()) + { + for (int index = 0; index < 4105; index++) + { + expected.Add(await write.Operations.EnqueueAsync(new CloudOperationJournalEntry( + Guid.NewGuid(), kinds[index % kinds.Length], null, BitConverter.GetBytes(index), + DateTimeOffset.UtcNow, attemptCount: index % 3))); + } + + await write.CommitAsync(); + } + + long through; + await using (ICloudStateTransaction capture = await store.BeginTransactionAsync()) + { + through = await Assert.IsAssignableFrom(capture.Operations) + .GetHighWaterSequenceAsync(); + await capture.RollbackAsync(); + } + + // Leave holes both before and inside later pages; the cursor must use assigned sequences, + // not an offset or the count of remaining rows. New rows must stay outside this scan. + await using (ICloudStateTransaction update = await store.BeginTransactionAsync()) + { + foreach (CloudOperationJournalEntry removed in expected.Where(row => row.Sequence % 17 == 0)) + { + await update.Operations.RemoveAsync(removed.OperationId); + } + + await update.Operations.EnqueueAsync(new CloudOperationJournalEntry( + Guid.NewGuid(), kinds[0], null, [99], DateTimeOffset.UtcNow)); + await update.CommitAsync(); + } + + expected.RemoveAll(row => row.Sequence % 17 == 0); + List delivered = []; + long cursor = 0; + bool more; + do + { + await using ICloudStateTransaction read = await store.BeginTransactionAsync(); + CloudOperationJournalPage page = await Assert.IsAssignableFrom(read.Operations) + .ReadPageAsync(cursor, through, 7); + Assert.InRange(page.Operations.Count, 1, 7); + Assert.All(page.Operations, row => Assert.InRange(row.Sequence, cursor + 1, through)); + Assert.Equal(page.Operations[^1].Sequence, page.LastScannedSequence); + delivered.AddRange(page.Operations); + cursor = page.LastScannedSequence; + more = page.HasMore; + await read.RollbackAsync(); + } + while (more); + + Assert.Equal(expected.Select(row => row.OperationId), delivered.Select(row => row.OperationId)); + for (int index = 0; index < expected.Count; index++) + { + Assert.Equal(expected[index].Sequence, delivered[index].Sequence); + Assert.Equal(expected[index].Kind, delivered[index].Kind); + Assert.Equal(expected[index].AttemptCount, delivered[index].AttemptCount); + Assert.Equal(expected[index].Payload.ToArray(), delivered[index].Payload.ToArray()); + } + + await using ICloudStateTransaction verify = await store.BeginTransactionAsync(); + ICloudOperationJournalPaging paging = Assert.IsAssignableFrom(verify.Operations); + CloudOperationJournalPage terminal = await paging.ReadPageAsync(through, through, 4096); + Assert.Empty(terminal.Operations); + Assert.Equal(through, terminal.LastScannedSequence); + Assert.False(terminal.HasMore); + Assert.Equal(expected.Count + 1, (await verify.Operations.ListAsync(int.MaxValue)).Count); + Assert.True(await paging.GetHighWaterSequenceAsync() > through); + await verify.RollbackAsync(); + } + + [Fact] + public async Task JournalPagingEnforcesBoundsCancellationAndTransactionLifetime() + { + await using ICloudStateStore store = await CreateFactory().OpenAsync(CreateContext()); + await using ICloudStateTransaction transaction = await store.BeginTransactionAsync(); + ICloudOperationJournalPaging paging = Assert.IsAssignableFrom(transaction.Operations); + Assert.Equal(0, await paging.GetHighWaterSequenceAsync()); + Assert.Empty((await paging.ReadPageAsync(0, 0, 1)).Operations); + foreach ((long after, long through, int limit) in new[] { (-1L, 0L, 1), (2L, 1L, 1), (0L, 0L, 0), (0L, 0L, 4097) }) + { + await Assert.ThrowsAsync(async () => + await paging.ReadPageAsync(after, through, limit)); + } + + using CancellationTokenSource cancellation = new(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(async () => + await paging.ReadPageAsync(0, 0, 1, cancellation.Token)); + await Assert.ThrowsAnyAsync(async () => + await paging.GetHighWaterSequenceAsync(cancellation.Token)); + await transaction.RollbackAsync(); + await Assert.ThrowsAsync(async () => await paging.ReadPageAsync(0, 0, 1)); + await Assert.ThrowsAsync(async () => await paging.GetHighWaterSequenceAsync()); + } + [Fact] public async Task CommittedItemSurvivesStoreReopen() { diff --git a/tests/CfSharp.Tests/Persistence/InMemoryCloudStateStoreContractTests.cs b/tests/CfSharp.Tests/Persistence/InMemoryCloudStateStoreContractTests.cs index f6192e6..2daf8c2 100644 --- a/tests/CfSharp.Tests/Persistence/InMemoryCloudStateStoreContractTests.cs +++ b/tests/CfSharp.Tests/Persistence/InMemoryCloudStateStoreContractTests.cs @@ -56,6 +56,7 @@ private sealed class InMemoryTransaction : ICloudItemStateRepository, ICloudCheckpointRepository, ICloudOperationJournal, + ICloudOperationJournalPaging, ICloudConflictRepository, ICloudRemoteBatchRepository, ICloudEchoSuppressionRepository @@ -248,6 +249,33 @@ public ValueTask> ListAsync( return ValueTask.FromResult(operations); } + public ValueTask GetHighWaterSequenceAsync(CancellationToken cancellationToken = default) + { + CheckActive(cancellationToken); + return ValueTask.FromResult(_state.Operations.Values.Select(row => row.Sequence).DefaultIfEmpty().Max()); + } + + public ValueTask ReadPageAsync( + long afterSequence, long throughSequence, int limit, CancellationToken cancellationToken = default) + { + CheckActive(cancellationToken); + ArgumentOutOfRangeException.ThrowIfNegative(afterSequence); + ArgumentOutOfRangeException.ThrowIfLessThan(throughSequence, afterSequence); + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(limit, 4096); + CloudOperationJournalEntry[] rows = _state.Operations.Values + .Where(row => row.Sequence > afterSequence && row.Sequence <= throughSequence) + .OrderBy(row => row.Sequence).Take(limit + 1).ToArray(); + bool more = rows.Length > limit; + if (more) + { + rows = rows[..limit]; + } + + return ValueTask.FromResult(new CloudOperationJournalPage(rows, + rows.Length == 0 ? throughSequence : rows[^1].Sequence, more)); + } + public ValueTask> ListByItemIdAsync( Guid itemId, int maximumCount, From 913ba7ce10986af0819e6528d7ebaf8054bf9a61 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:51:28 +0800 Subject: [PATCH 02/40] feat(local-changes): expose bounded typed change scans --- docs/local-change-feed.md | 42 ++++ eng/api-baseline.json | 14 ++ .../CloudLocalChangeFeed.Scans.cs | 114 ++++++++++ .../LocalChanges/CloudLocalChangeFeed.cs | 2 +- .../LocalChanges/CloudLocalChangeScan.cs | 85 ++++++++ .../CloudLocalChangeFeedTests.cs | 2 +- .../CloudLocalChangeScanTests.cs | 205 ++++++++++++++++++ 7 files changed, 462 insertions(+), 2 deletions(-) create mode 100644 src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs create mode 100644 src/CfSharp/LocalChanges/CloudLocalChangeScan.cs create mode 100644 tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanTests.cs diff --git a/docs/local-change-feed.md b/docs/local-change-feed.md index 69e368b..e5f95af 100644 --- a/docs/local-change-feed.md +++ b/docs/local-change-feed.md @@ -28,6 +28,48 @@ The feed normalizes paths relative to the sync root, pairs renames, and persists journal with its watcher checkpoint in one transaction. Acknowledgement advances durable progress only after the application has accepted the batch. +## Scan a finite backlog without acknowledging its head + +Use typed scans when some pending changes cannot currently be dispatched. A scan captures the +highest pending sequence once; each page uses an exclusive lower bound and that inclusive upper +bound. Reads never ACK a deferred row, wait for a new event, or extend the scan with later writes. + +```csharp +CloudLocalChangeScan scan = await feed.BeginScanAsync(cancellationToken); +long after = 0; +while (true) +{ + CloudLocalChangePage page = await feed.ReadPageAsync(scan, after, 4, cancellationToken); + if (page.RequiresFullRescan) + { + // Stop dispatch. Replay pending remote creations, reconcile, explicitly acknowledge + // the rescan, then begin a new scan from zero. + break; + } + + await DispatchEligibleChangesAsync(page.Changes, cancellationToken); + // ACK only operation IDs whose meaning the application has durably accepted. + after = page.LastScannedSequence; + if (!page.HasMore) + { + break; + } +} +``` + +The page limit is 1 through 4096 and is independent of the legacy `BatchSize` (default 64). +Use `LastScannedSequence` directly, without adding one. ACKs between pages can leave sequence +gaps and can make the next page empty. Empty terminal pages return `ThroughSequence` immediately. +Restarting the feed or runtime invalidates old scans; create a new scan from zero against the same +durable store. `StoreScope` identifies the running feed lifetime, not a persisted database identity. +Scans and pages retain no transaction, reader, native handle, or file content. + +Custom stores opt in by implementing `ICloudOperationJournalPaging` on `Operations`; unsupported +stores throw `NotSupportedException` for scans and retain the existing batch API. Invalid or foreign +journal payloads throw `CloudLocalChangeJournalException` with the original operation ID and +sequence, leaving the row pending. Consumers do not need private payload parsing or a copied queue. +Scheduling across roots and preserving dependencies between changes remain application concerns. + ## Rescan is a normal state A buffer overflow, watcher error, or ambiguous rename is reported as `RequiresFullRescan`. Stop diff --git a/eng/api-baseline.json b/eng/api-baseline.json index 9b30f69..f633a22 100644 --- a/eng/api-baseline.json +++ b/eng/api-baseline.json @@ -1062,6 +1062,8 @@ "method System.Threading.Tasks.ValueTask SuppressProviderEchoAsync(CfSharp.CloudStateOperationKind kind,System.String relativePath,System.DateTimeOffset expiresAt,System.Nullable\u003CSystem.Guid\u003E itemId,System.String previousRelativePath,System.Int32 expectedObservationCount,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask SuppressProviderEchoAsync(CfSharp.CloudStateOperationKind kind,System.String relativePath,System.DateTimeOffset expiresAt,System.Nullable\u003CSystem.Guid\u003E itemId,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudLocalChangeBatch\u003E ReadBatchAsync(System.Threading.CancellationToken cancellationToken)", + "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudLocalChangePage\u003E ReadPageAsync(CfSharp.CloudLocalChangeScan scan,System.Int64 afterSequence,System.Int32 limit,System.Threading.CancellationToken cancellationToken)", + "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudLocalChangeScan\u003E BeginScanAsync(System.Threading.CancellationToken cancellationToken)", "method System.Void Dispose()", "property System.Boolean IsStarted()", "property System.String SyncRootPath()", @@ -1076,6 +1078,9 @@ "property System.Int32 BufferCapacity()", "property CfSharp.CloudLocalChangeFeedOptions Default()", "property System.TimeSpan ShutdownTimeout()", + "type CfSharp.CloudLocalChangeJournalException", + "property System.Guid OperationId()", + "property System.Int64 Sequence()", "type CfSharp.CloudLocalChangeKind", "field CfSharp.CloudLocalChangeKind ContentUpdate", "field CfSharp.CloudLocalChangeKind Create", @@ -1083,6 +1088,15 @@ "field CfSharp.CloudLocalChangeKind MetadataUpdate", "field CfSharp.CloudLocalChangeKind Move", "field System.Int32 value__", + "type CfSharp.CloudLocalChangePage", + "property System.Collections.Generic.IReadOnlyList\u003CCfSharp.CloudLocalChange\u003E Changes()", + "property System.Boolean HasMore()", + "property System.Int64 LastScannedSequence()", + "property System.Boolean RequiresFullRescan()", + "type CfSharp.CloudLocalChangeScan", + "property System.Boolean RequiresFullRescan()", + "property System.Guid StoreScope()", + "property System.Int64 ThroughSequence()", "type CfSharp.CloudLocalFileBinding", "ctor CfSharp.CloudLocalFileBinding(System.UInt64 volumeSerialNumber,System.Guid syncRootFileId,System.Guid localFileId)", "method CfSharp.CloudLocalFileBinding \u003CClone\u003E$()", diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs new file mode 100644 index 0000000..84bab27 --- /dev/null +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs @@ -0,0 +1,114 @@ +namespace CfSharp; + +public sealed partial class CloudLocalChangeFeed +{ + private readonly Guid _scanScope = Guid.NewGuid(); + + /// Captures a finite boundary for non-waiting, bounded reads of pending local changes. + /// Token that cancels the read without changing durable state. + /// A handle-free scan owned by this running feed, including its reconciliation status. + /// + /// Journal and recovery markers are read in one short transaction. Later enqueues lie outside + /// the captured boundary. Call ReadPageAsync with zero initially; restarting requires a new + /// scan. This Windows feed can have concurrent consumers; callers own their dispatch and ACK + /// coordination. Capturing a scan does not wait for native notifications still being processed. + /// + /// The custom journal does not implement optional paging. + /// The feed is not started or its worker failed. + /// The feed has been disposed. + /// The read was canceled. + public async ValueTask BeginScanAsync(CancellationToken cancellationToken = default) + { + EnsureStarted(); + ThrowIfFailed(); + await using ICloudStateTransaction transaction = await _stateStore.BeginTransactionAsync(cancellationToken) + .ConfigureAwait(false); + ICloudOperationJournalPaging paging = RequirePaging(transaction); + long through = await paging.GetHighWaterSequenceAsync(cancellationToken).ConfigureAwait(false); + bool rescan = await RequiresScanReconciliationAsync(transaction, cancellationToken).ConfigureAwait(false); + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + return new CloudLocalChangeScan(_scanScope, through, rescan); + } + + /// Reads one typed page inside a previously captured scan, without waiting or acknowledging. + /// The scan created by this feed during its current lifetime. + /// Exclusive lower bound, initially zero, then the preceding page's cursor. + /// Maximum delivered changes, from one through 4096, independent of BatchSize. + /// Token that cancels the query without advancing durable progress. + /// An owned page. Empty terminal pages return the scan boundary with HasMore false. + /// + /// Reads preserve original journal identifiers, nullable item associations, sequences, paths, + /// and observation times. An ACK between pages may remove rows; new writes never extend this + /// scan. Stop on RequiresFullRescan even if HasMore is true. No remote-provider code is invoked. + /// + /// The scan is null. + /// The scan belongs to a different feed or runtime. + /// The cursor is outside the scan or the limit is invalid. + /// The journal lacks the optional paging capability. + /// An official local change cannot be decoded; its row remains pending. + /// The feed is not started or its worker failed. + /// The feed has been disposed. + /// The read was canceled. + public async ValueTask ReadPageAsync(CloudLocalChangeScan scan, long afterSequence, + int limit, CancellationToken cancellationToken = default) + { + EnsureStarted(); + ThrowIfFailed(); + ArgumentNullException.ThrowIfNull(scan); + if (scan.StoreScope != _scanScope) + { + throw new ArgumentException("The scan belongs to another feed lifetime. Begin a new scan on this feed.", nameof(scan)); + } + + ArgumentOutOfRangeException.ThrowIfNegative(afterSequence); + ArgumentOutOfRangeException.ThrowIfGreaterThan(afterSequence, scan.ThroughSequence); + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(limit, 4096); + await using ICloudStateTransaction transaction = await _stateStore.BeginTransactionAsync(cancellationToken) + .ConfigureAwait(false); + CloudOperationJournalPage journalPage = await RequirePaging(transaction) + .ReadPageAsync(afterSequence, scan.ThroughSequence, limit, cancellationToken).ConfigureAwait(false); + bool rescan = scan.RequiresFullRescan || + await RequiresScanReconciliationAsync(transaction, cancellationToken).ConfigureAwait(false); + CloudLocalChange[] changes = rescan ? [] : journalPage.Operations.Select(DecodeScannedChange).ToArray(); + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + return new CloudLocalChangePage(changes, rescan ? afterSequence : journalPage.LastScannedSequence, + rescan ? journalPage.Operations.Count != 0 : journalPage.HasMore, rescan); + } + + private CloudLocalChange DecodeScannedChange(CloudOperationJournalEntry operation) + { + try + { + CloudLocalChange change = ToChange(operation); + CloudItemPathResolver.Resolve(_syncRootPath, change.RelativePath, allowRoot: false); + if (change.PreviousRelativePath is not null) + { + CloudItemPathResolver.Resolve(_syncRootPath, change.PreviousRelativePath, allowRoot: false); + } + + if ((change.Kind == CloudLocalChangeKind.Move) != (change.PreviousRelativePath is not null)) + { + throw new InvalidOperationException("The local-change move paths are inconsistent."); + } + + return change; + } + catch (Exception exception) when (exception is InvalidOperationException or ArgumentException) + { + throw new CloudLocalChangeJournalException(operation, exception); + } + } + + private static ICloudOperationJournalPaging RequirePaging(ICloudStateTransaction transaction) => + transaction.Operations as ICloudOperationJournalPaging ?? + throw new NotSupportedException("This state store does not implement ICloudOperationJournalPaging. Legacy batch reads remain available."); + + private static async ValueTask RequiresScanReconciliationAsync(ICloudStateTransaction transaction, + CancellationToken cancellationToken) => + (await ReadCheckpointAsync(transaction, cancellationToken).ConfigureAwait(false)).RequiresFullRescan || + (await transaction.Checkpoints.ListAsync(RemoteCreationIntent.ObservationsPrefix, cancellationToken) + .ConfigureAwait(false)).Count != 0 || + (await transaction.Checkpoints.ListAsync(RemoteCreationIntent.Prefix, cancellationToken) + .ConfigureAwait(false)).Any(value => !RemoteCreationIntent.Decode(value.Value).Committed); +} diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs index ccc6f8c..420f7df 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs @@ -19,7 +19,7 @@ namespace CfSharp; /// intentionally outside CfSharp. /// /// -public sealed class CloudLocalChangeFeed : IDisposable, IAsyncDisposable +public sealed partial class CloudLocalChangeFeed : IDisposable, IAsyncDisposable { internal const string CheckpointName = "cfsharp.local-change-feed.v1"; diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeScan.cs b/src/CfSharp/LocalChanges/CloudLocalChangeScan.cs new file mode 100644 index 0000000..e08f21d --- /dev/null +++ b/src/CfSharp/LocalChanges/CloudLocalChangeScan.cs @@ -0,0 +1,85 @@ +using System.Collections.ObjectModel; + +namespace CfSharp; + +/// Retains a finite journal boundary belonging to one running local-change feed. +/// +/// This immutable, concurrently readable value owns no transaction, reader, watcher, or native +/// handle. It is valid only with the feed that created it during that feed's current lifetime. +/// Retaining or serializing its scalar properties does not create a resumable cross-process scan. +/// After restarting, begin a new scan from zero against the durable pending journal. +/// +public sealed class CloudLocalChangeScan +{ + internal CloudLocalChangeScan(Guid storeScope, long throughSequence, bool requiresFullRescan) + { + StoreScope = storeScope; + ThroughSequence = throughSequence; + RequiresFullRescan = requiresFullRescan; + } + + /// Gets the opaque identity of this feed's running store-consumer lifetime. + /// This value is not a durable database identity or native object binding. + public Guid StoreScope { get; } + + /// Gets the inclusive highest pending sequence captured when the scan began. + public long ThroughSequence { get; } + + /// Gets whether reconciliation was required when this scan began. + /// Pages also check for a reconciliation condition that arose after capture. + public bool RequiresFullRescan { get; } +} + +/// Contains an owned, bounded typed journal page and its continuation facts. +/// +/// The immutable collection is safe for concurrent reads after the originating transaction ends. +/// A page never acknowledges operations. When reconciliation is required, changes are withheld +/// and the cursor does not advance; stop scanning, reconcile, acknowledge the rescan, and start +/// a new scan. HasMore then describes pending rows, not permission to dispatch them. +/// +public sealed class CloudLocalChangePage +{ + internal CloudLocalChangePage(IReadOnlyList changes, long lastScannedSequence, + bool hasMore, bool requiresFullRescan) + { + Changes = new ReadOnlyCollection(changes.ToArray()); + LastScannedSequence = lastScannedSequence; + HasMore = hasMore; + RequiresFullRescan = requiresFullRescan; + } + + /// Gets the immutable changes in strictly ascending durable sequence order. + public IReadOnlyList Changes { get; } + + /// Gets the exclusive lower bound for the next read; use it without adding one. + public long LastScannedSequence { get; } + + /// Gets whether an undelivered row remained in this scan's range during the read. + /// Concurrent acknowledgements may make the next page empty. + public bool HasMore { get; } + + /// Gets whether dispatch must stop until full reconciliation and a new scan. + public bool RequiresFullRescan { get; } +} + +/// Identifies a durable operation that cannot be decoded as an official local change. +/// +/// The offending operation remains pending with its original payload, identifier, and sequence. +/// This exception owns no storage resources and preserves the decoding failure as InnerException. +/// Do not acknowledge the row merely to continue scanning; repair or reconcile its meaning first. +/// +public sealed class CloudLocalChangeJournalException : InvalidOperationException +{ + internal CloudLocalChangeJournalException(CloudOperationJournalEntry operation, Exception innerException) + : base($"Journal operation {operation.OperationId:D} at sequence {operation.Sequence} is not a valid local change.", innerException) + { + OperationId = operation.OperationId; + Sequence = operation.Sequence; + } + + /// Gets the unchanged identifier of the operation that failed decoding. + public Guid OperationId { get; } + + /// Gets the unchanged durable sequence of that operation. + public long Sequence { get; } +} diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs index e4dbfb6..56b2120 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs @@ -1,6 +1,6 @@ namespace CfSharp.Storage.Sqlite.Tests; -public sealed class CloudLocalChangeFeedTests : IAsyncLifetime +public sealed partial class CloudLocalChangeFeedTests : IAsyncLifetime { private string _temporaryDirectory = null!; private string _syncRootPath = null!; diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanTests.cs new file mode 100644 index 0000000..056784c --- /dev/null +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanTests.cs @@ -0,0 +1,205 @@ +namespace CfSharp.Storage.Sqlite.Tests; + +public sealed partial class CloudLocalChangeFeedTests +{ + [Fact] + public async Task CustomStoreWithoutPagingRetainsLegacyBatchReads() + { + await using ICloudStateStore store = await OpenStoreAsync(); + CloudOperationJournalEntry expected = Assert.Single(await SeedChangesAsync(store, 1)); + await using CloudLocalChangeFeed feed = CreateFeed(new LegacyStore(store), new FakeSource()); + await feed.StartAsync(); + await Assert.ThrowsAsync(async () => await feed.BeginScanAsync()); + Assert.Equal(expected.OperationId, Assert.Single((await feed.ReadBatchAsync()).Changes).OperationId); + await feed.AcknowledgeAsync([expected.OperationId]); + } + + [Fact] + public async Task ScanReachesSuccessorBehindDeferredHeadAndReplaysAfterStoreRestart() + { + CloudLocalChangeScan previousScan; + CloudOperationJournalEntry[] seed; + await using (ICloudStateStore store = await OpenStoreAsync()) + await using (CloudLocalChangeFeed feed = CreateFeed(store, new FakeSource())) + { + seed = await SeedChangesAsync(store, 17); + await feed.StartAsync(); + previousScan = await feed.BeginScanAsync(); + Assert.Equal(seed[^1].Sequence, previousScan.ThroughSequence); + CloudLocalChangePage first = await feed.ReadPageAsync(previousScan, 0, 4); + CloudLocalChangePage repeated = await feed.ReadPageAsync(previousScan, 0, 4); + Assert.Equal(first.Changes.Select(value => value.OperationId), repeated.Changes.Select(value => value.OperationId)); + Assert.Equal(4, first.Changes.Count); + + CloudOperationJournalEntry later = Assert.Single(await SeedChangesAsync(store, 1)); + List delivered = []; + long cursor = 0; + CloudLocalChangePage page; + do + { + page = await feed.ReadPageAsync(previousScan, cursor, 4); + Assert.False(page.RequiresFullRescan); + delivered.AddRange(page.Changes); + cursor = page.LastScannedSequence; + } + while (page.HasMore); + + Assert.Equal(seed.Select(row => row.OperationId), delivered.Select(value => value.OperationId)); + Assert.DoesNotContain(delivered, value => value.OperationId == later.OperationId); + Assert.Equal(seed.Select(row => row.Sequence), delivered.Select(value => value.Sequence)); + Assert.Equal("moved-3.txt", delivered[3].RelativePath); + Assert.Equal("old-3.txt", delivered[3].PreviousRelativePath); + Assert.Equal(seed[3].CreatedAt, delivered[3].ObservedAt); + await feed.AcknowledgeAsync([delivered[^1].OperationId]); + CloudLocalChangePage terminal = await feed.ReadPageAsync(previousScan, cursor, 4); + Assert.Empty(terminal.Changes); + Assert.False(terminal.HasMore); + Assert.Equal(previousScan.ThroughSequence, terminal.LastScannedSequence); + + await using ICloudStateTransaction verify = await store.BeginTransactionAsync(); + Assert.Equal(seed.Take(16).Select(row => row.OperationId), + (await verify.Operations.ListAsync(16)).Select(row => row.OperationId)); + await verify.RollbackAsync(); + } + + await using ICloudStateStore reopened = await OpenStoreAsync(); + await using CloudLocalChangeFeed restarted = CreateFeed(reopened, new FakeSource()); + await restarted.StartAsync(); + await Assert.ThrowsAsync(async () => await restarted.ReadPageAsync(previousScan, 0, 4)); + CloudLocalChangeScan fresh = await restarted.BeginScanAsync(); + Assert.NotEqual(previousScan.StoreScope, fresh.StoreScope); + Assert.Equal(seed.Take(4).Select(row => row.OperationId), + (await restarted.ReadPageAsync(fresh, 0, 4)).Changes.Select(value => value.OperationId)); + } + + [Fact] + public async Task ScanHandlesAcknowledgementHolesAndRejectsInvalidConsumers() + { + await using ICloudStateStore store = await OpenStoreAsync(); + CloudOperationJournalEntry[] seed = await SeedChangesAsync(store, 9); + await using CloudLocalChangeFeed feed = CreateFeed(store, new FakeSource()); + await Assert.ThrowsAsync(async () => await feed.BeginScanAsync()); + await feed.StartAsync(); + CloudLocalChangeScan scan = await feed.BeginScanAsync(); + CloudLocalChangePage first = await feed.ReadPageAsync(scan, 0, 4); + await feed.AcknowledgeAsync(seed.Skip(4).Take(4).Select(row => row.OperationId)); + CloudLocalChangePage second = await feed.ReadPageAsync(scan, first.LastScannedSequence, 4); + Assert.Equal(seed[^1].OperationId, Assert.Single(second.Changes).OperationId); + Assert.False(second.HasMore); + + await using CloudLocalChangeFeed another = CreateFeed(store, new FakeSource()); + await another.StartAsync(); + await Assert.ThrowsAsync(async () => await another.ReadPageAsync(scan, 0, 4)); + await Assert.ThrowsAsync(async () => await feed.ReadPageAsync(scan, -1, 4)); + await Assert.ThrowsAsync(async () => await feed.ReadPageAsync(scan, scan.ThroughSequence + 1, 4)); + await Assert.ThrowsAsync(async () => await feed.ReadPageAsync(scan, 0, 4097)); + using CancellationTokenSource canceled = new(); + canceled.Cancel(); + await Assert.ThrowsAnyAsync(async () => await feed.ReadPageAsync(scan, 0, 4, canceled.Token)); + Assert.NotEmpty((await feed.ReadPageAsync(scan, 0, 4)).Changes); + await feed.DisposeAsync(); + await Assert.ThrowsAsync(async () => await feed.BeginScanAsync()); + } + + [Fact] + public async Task EmptyScanReturnsImmediatelyAndDoesNotExpandAfterEnqueue() + { + await using ICloudStateStore store = await OpenStoreAsync(); + await using CloudLocalChangeFeed feed = CreateFeed(store, new FakeSource()); + await feed.StartAsync(); + CloudLocalChangeScan empty = await feed.BeginScanAsync(); + Assert.Equal(0, empty.ThroughSequence); + await SeedChangesAsync(store, 1); + CloudLocalChangePage page = await feed.ReadPageAsync(empty, 0, 1); + Assert.Empty(page.Changes); + Assert.False(page.HasMore); + Assert.Equal(0, page.LastScannedSequence); + Assert.Single((await feed.ReadPageAsync(await feed.BeginScanAsync(), 0, 1)).Changes); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MalformedOrForeignPayloadRetainsOffendingJournalRow(bool foreignPath) + { + await using ICloudStateStore store = await OpenStoreAsync(); + Guid operationId = Guid.NewGuid(); + byte[] payload = foreignPath + ? new LocalChangePayload("..\\outside.txt", null, false, DateTimeOffset.UtcNow).Encode() + : [0xff, 0xff, 0xff, 0xff, 0x7f]; + await using (ICloudStateTransaction write = await store.BeginTransactionAsync()) + { + await write.Operations.EnqueueAsync(new CloudOperationJournalEntry(operationId, + CloudStateOperationKind.Create, null, payload, DateTimeOffset.UtcNow)); + await write.CommitAsync(); + } + + await using CloudLocalChangeFeed feed = CreateFeed(store, new FakeSource()); + await feed.StartAsync(); + CloudLocalChangeScan scan = await feed.BeginScanAsync(); + CloudLocalChangeJournalException failure = await Assert.ThrowsAsync( + async () => await feed.ReadPageAsync(scan, 0, 1)); + Assert.Equal(operationId, failure.OperationId); + Assert.Equal(1, failure.Sequence); + Assert.NotNull(failure.InnerException); + await using ICloudStateTransaction verify = await store.BeginTransactionAsync(); + Assert.Equal(payload, (await verify.Operations.GetAsync(operationId))!.Payload.ToArray()); + await verify.RollbackAsync(); + } + + private static async Task SeedChangesAsync(ICloudStateStore store, int count) + { + await using ICloudStateTransaction transaction = await store.BeginTransactionAsync(); + List rows = []; + for (int index = 0; index < count; index++) + { + CloudStateOperationKind kind = (CloudStateOperationKind)(index % 5); + DateTimeOffset observed = DateTimeOffset.UtcNow.AddTicks(index); + string path = kind == CloudStateOperationKind.Move ? $"moved-{index}.txt" : $"item-{index}.txt"; + LocalChangePayload payload = new(path, + kind == CloudStateOperationKind.Move ? $"old-{index}.txt" : null, false, observed); + rows.Add(await transaction.Operations.EnqueueAsync(new CloudOperationJournalEntry( + Guid.NewGuid(), kind, null, payload.Encode(), observed))); + } + + await transaction.CommitAsync(); + return rows.ToArray(); + } + + private sealed class LegacyStore(ICloudStateStore inner) : ICloudStateStore + { + public async ValueTask BeginTransactionAsync(CancellationToken cancellationToken = default) => + new LegacyTransaction(await inner.BeginTransactionAsync(cancellationToken)); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class LegacyTransaction(ICloudStateTransaction inner) : ICloudStateTransaction + { + public ICloudItemStateRepository Items => inner.Items; + public ICloudCheckpointRepository Checkpoints => inner.Checkpoints; + public ICloudOperationJournal Operations { get; } = new LegacyJournal(inner.Operations); + public ICloudConflictRepository Conflicts => inner.Conflicts; + public ICloudRemoteBatchRepository RemoteBatches => inner.RemoteBatches; + public ICloudEchoSuppressionRepository EchoSuppressions => inner.EchoSuppressions; + public ValueTask CommitAsync(CancellationToken cancellationToken = default) => inner.CommitAsync(cancellationToken); + public ValueTask RollbackAsync(CancellationToken cancellationToken = default) => inner.RollbackAsync(cancellationToken); + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + + private sealed class LegacyJournal(ICloudOperationJournal inner) : ICloudOperationJournal + { + public ValueTask EnqueueAsync(CloudOperationJournalEntry operation, + CancellationToken cancellationToken = default) => inner.EnqueueAsync(operation, cancellationToken); + public ValueTask GetAsync(Guid operationId, + CancellationToken cancellationToken = default) => inner.GetAsync(operationId, cancellationToken); + public ValueTask> ListAsync(int maximumCount, + CancellationToken cancellationToken = default) => inner.ListAsync(maximumCount, cancellationToken); + public ValueTask> ListByItemIdAsync(Guid itemId, int maximumCount, + CancellationToken cancellationToken = default) => inner.ListByItemIdAsync(itemId, maximumCount, cancellationToken); + public ValueTask UpdateAsync(CloudOperationJournalEntry operation, + CancellationToken cancellationToken = default) => inner.UpdateAsync(operation, cancellationToken); + public ValueTask RemoveAsync(Guid operationId, + CancellationToken cancellationToken = default) => inner.RemoveAsync(operationId, cancellationToken); + } +} From 47b6048d102804c5c68ab1a3484988a8c31068f0 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:03:11 +0800 Subject: [PATCH 03/40] fix(local-changes): preserve reconciliation fences during scans --- docs/local-change-feed.md | 11 + .../CloudLocalChangeFeed.Scans.cs | 40 +++- .../LocalChanges/CloudLocalChangeFeed.cs | 89 ++++---- .../LocalChanges/CloudLocalChangeScan.cs | 8 +- .../RemoteCreationRecoveryTests.cs | 6 + .../CloudLocalChangeFeedTests.cs | 4 +- .../CloudLocalChangeScanFenceTests.cs | 196 ++++++++++++++++++ 7 files changed, 298 insertions(+), 56 deletions(-) create mode 100644 tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs diff --git a/docs/local-change-feed.md b/docs/local-change-feed.md index e5f95af..8c33ff4 100644 --- a/docs/local-change-feed.md +++ b/docs/local-change-feed.md @@ -76,6 +76,17 @@ A buffer overflow, watcher error, or ambiguous rename is reported as `RequiresFu assuming that notifications are complete, enumerate the materialized local tree, reconcile it, and acknowledge the rescan explicitly. Recursive operations never follow links or remote children. +Scans and legacy batches both withhold dispatchable changes while recovery is required, including +when a pending backlog already exists. Pending remote creation intents and their uncertain watcher +observations are checked in the same transaction as each page. Replay creation recovery before +acknowledging reconciliation. A loss received while the watcher worker waits for the store also +blocks pages immediately. Reading a page never clears recovery markers. + +An overflow/error or a full-rescan acknowledgement invalidates previously captured scans even +after the durable marker is cleared. Begin a new scan after recovery. If a new loss races +`AcknowledgeFullRescanAsync`, it either rejects acknowledgement with `InvalidOperationException` +or remains observable as a new rescan condition; finish reconciliation before retrying. + ## Provider echoes Provider-originated writes can be wrapped by `SuppressProviderEchoAsync` so they do not become diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs index 84bab27..1c244d8 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs @@ -3,6 +3,9 @@ namespace CfSharp; public sealed partial class CloudLocalChangeFeed { private readonly Guid _scanScope = Guid.NewGuid(); + private long _lossGeneration; + private long _persistedLossGeneration; + private long _rescanAcknowledgementGeneration; /// Captures a finite boundary for non-waiting, bounded reads of pending local changes. /// Token that cancels the read without changing durable state. @@ -21,13 +24,18 @@ public async ValueTask BeginScanAsync(CancellationToken ca { EnsureStarted(); ThrowIfFailed(); + long generation = Volatile.Read(ref _lossGeneration); + long acknowledgementGeneration = Volatile.Read(ref _rescanAcknowledgementGeneration); await using ICloudStateTransaction transaction = await _stateStore.BeginTransactionAsync(cancellationToken) .ConfigureAwait(false); ICloudOperationJournalPaging paging = RequirePaging(transaction); long through = await paging.GetHighWaterSequenceAsync(cancellationToken).ConfigureAwait(false); bool rescan = await RequiresScanReconciliationAsync(transaction, cancellationToken).ConfigureAwait(false); await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); - return new CloudLocalChangeScan(_scanScope, through, rescan); + ThrowIfFailed(); + rescan |= HasUnpersistedLoss || generation != Volatile.Read(ref _lossGeneration) || + acknowledgementGeneration != Volatile.Read(ref _rescanAcknowledgementGeneration); + return new CloudLocalChangeScan(_scanScope, through, rescan, generation, acknowledgementGeneration); } /// Reads one typed page inside a previously captured scan, without waiting or acknowledging. @@ -68,10 +76,18 @@ public async ValueTask ReadPageAsync(CloudLocalChangeScan .ConfigureAwait(false); CloudOperationJournalPage journalPage = await RequirePaging(transaction) .ReadPageAsync(afterSequence, scan.ThroughSequence, limit, cancellationToken).ConfigureAwait(false); - bool rescan = scan.RequiresFullRescan || + bool rescan = scan.RequiresFullRescan || HasUnpersistedLoss || + ScanIsInvalidated(scan) || await RequiresScanReconciliationAsync(transaction, cancellationToken).ConfigureAwait(false); CloudLocalChange[] changes = rescan ? [] : journalPage.Operations.Select(DecodeScannedChange).ToArray(); await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + ThrowIfFailed(); + rescan |= HasUnpersistedLoss || ScanIsInvalidated(scan); + if (rescan) + { + changes = []; + } + return new CloudLocalChangePage(changes, rescan ? afterSequence : journalPage.LastScannedSequence, rescan ? journalPage.Operations.Count != 0 : journalPage.HasMore, rescan); } @@ -104,9 +120,29 @@ private static ICloudOperationJournalPaging RequirePaging(ICloudStateTransaction transaction.Operations as ICloudOperationJournalPaging ?? throw new NotSupportedException("This state store does not implement ICloudOperationJournalPaging. Legacy batch reads remain available."); + private bool HasUnpersistedLoss => + Volatile.Read(ref _lossGeneration) != Volatile.Read(ref _persistedLossGeneration); + + private bool ScanIsInvalidated(CloudLocalChangeScan scan) => + scan.LossGeneration != Volatile.Read(ref _lossGeneration) || + scan.AcknowledgementGeneration != Volatile.Read(ref _rescanAcknowledgementGeneration); + + private long SignalLoss() + { + // Record loss before waiting for the store gate. A page may otherwise dispatch a backlog + // while the worker is still blocked from persisting the corresponding marker. Generations + // also invalidate older scans after explicit rescan acknowledgement clears the marker. + Volatile.Write(ref _rescanNoticeDelivered, 0); + return Interlocked.Increment(ref _lossGeneration); + } + private static async ValueTask RequiresScanReconciliationAsync(ICloudStateTransaction transaction, CancellationToken cancellationToken) => (await ReadCheckpointAsync(transaction, cancellationToken).ConfigureAwait(false)).RequiresFullRescan || + await RequiresCreationReconciliationAsync(transaction, cancellationToken).ConfigureAwait(false); + + private static async ValueTask RequiresCreationReconciliationAsync(ICloudStateTransaction transaction, + CancellationToken cancellationToken) => (await transaction.Checkpoints.ListAsync(RemoteCreationIntent.ObservationsPrefix, cancellationToken) .ConfigureAwait(false)).Count != 0 || (await transaction.Checkpoints.ListAsync(RemoteCreationIntent.Prefix, cancellationToken) diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs index 420f7df..91dffa8 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs @@ -126,26 +126,25 @@ public async ValueTask ReadBatchAsync( while (true) { ThrowIfFailed(); - if (await RequiresCreationReconciliationAsync(cancellationToken).ConfigureAwait(false)) + await using ICloudStateTransaction transaction = await _stateStore + .BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + LocalChangeCheckpoint checkpoint = await ReadCheckpointAsync(transaction, cancellationToken) + .ConfigureAwait(false); + bool creation = await RequiresCreationReconciliationAsync(transaction, cancellationToken).ConfigureAwait(false); + IReadOnlyList operations = await transaction.Operations + .ListAsync(_options.BatchSize, cancellationToken).ConfigureAwait(false); + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + ThrowIfFailed(); + bool rescan = creation || checkpoint.RequiresFullRescan || HasUnpersistedLoss; + if (rescan && (creation || operations.Count != 0 || + Interlocked.CompareExchange(ref _rescanNoticeDelivered, 1, 0) == 0)) { return new CloudLocalChangeBatch([], requiresFullRescan: true); } - IReadOnlyList operations = await ListOperationsAsync( - cancellationToken).ConfigureAwait(false); - if (operations.Count != 0) - { - return new CloudLocalChangeBatch( - operations.Select(ToChange).ToArray(), - requiresFullRescan: false); - } - - LocalChangeCheckpoint checkpoint = await ReadCheckpointAsync(cancellationToken) - .ConfigureAwait(false); - if (checkpoint.RequiresFullRescan && - Interlocked.CompareExchange(ref _rescanNoticeDelivered, 1, 0) == 0) + if (!rescan && operations.Count != 0) { - return new CloudLocalChangeBatch([], requiresFullRescan: true); + return new CloudLocalChangeBatch(operations.Select(ToChange).ToArray(), requiresFullRescan: false); } await _availability.Reader.ReadAsync(cancellationToken).ConfigureAwait(false); @@ -239,9 +238,14 @@ await transaction.Operations.RemoveAsync(acknowledgement.OperationId, cancellati /// reconciliation of the sync root. /// /// Token that cancels before the marker commit. + /// + /// A creation remains pending, a loss signal is not yet durable, or new loss raced this acknowledgement. + /// Finish recovery and retry after the worker has persisted its signal. + /// public async ValueTask AcknowledgeFullRescanAsync(CancellationToken cancellationToken = default) { EnsureStarted(); + long generation = Volatile.Read(ref _lossGeneration); await using ICloudStateTransaction transaction = await _stateStore .BeginTransactionAsync(cancellationToken).ConfigureAwait(false); IReadOnlyList creations = await transaction.Checkpoints @@ -260,6 +264,11 @@ public async ValueTask AcknowledgeFullRescanAsync(CancellationToken cancellation LocalChangeCheckpoint checkpoint = await ReadCheckpointAsync( transaction, cancellationToken).ConfigureAwait(false); + if (HasUnpersistedLoss || generation != Volatile.Read(ref _lossGeneration)) + { + throw new InvalidOperationException("A new loss signal must be persisted and reconciled before acknowledging the rescan."); + } + await transaction.Checkpoints.UpsertAsync( new CloudStateCheckpoint( CheckpointName, @@ -267,6 +276,10 @@ await transaction.Checkpoints.UpsertAsync( DateTimeOffset.UtcNow), cancellationToken).ConfigureAwait(false); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + // Full reconciliation establishes a new consumer boundary even when pending creation + // recovery cleared its records before this feed read them. Pre-reconciliation scans must + // not resume after an acknowledgement makes the durable marker disappear. + Interlocked.Increment(ref _rescanAcknowledgementGeneration); Volatile.Write(ref _rescanNoticeDelivered, 0); } @@ -628,6 +641,7 @@ private async ValueTask PersistChangeAsync( string.Equals(intent.RelativePath, previousPath?.RelativePath, StringComparison.OrdinalIgnoreCase))); if (pendingCreation) { + long generation = SignalLoss(); // Do not assign a local identity while native creation is uncommitted. Retain // the observation durably and require reconciliation after remote replay; its // origin cannot be proved solely from a path, timestamp, or notification kind. @@ -638,6 +652,7 @@ await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint( uncertain.Encode(), observedAt), cancellationToken).ConfigureAwait(false); await UpsertCheckpointAsync(transaction, observation, true, observedAt, cancellationToken).ConfigureAwait(false); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + Volatile.Write(ref _persistedLossGeneration, generation); SignalAvailable(); return; } @@ -759,20 +774,6 @@ await UpsertCheckpointAsync( SignalAvailable(); } - private async ValueTask RequiresCreationReconciliationAsync(CancellationToken cancellationToken) - { - await using ICloudStateTransaction transaction = await _stateStore.BeginTransactionAsync(cancellationToken) - .ConfigureAwait(false); - if ((await transaction.Checkpoints.ListAsync(RemoteCreationIntent.ObservationsPrefix, cancellationToken) - .ConfigureAwait(false)).Count != 0) - { - return true; - } - - return (await transaction.Checkpoints.ListAsync(RemoteCreationIntent.Prefix, cancellationToken) - .ConfigureAwait(false)).Any(value => !RemoteCreationIntent.Decode(value.Value).Committed); - } - private static bool MatchesRemoteCreationEcho(CloudEchoSuppressionState suppression, CloudStateOperationKind kind, CloudItemPath path, CloudItemState? state) { @@ -796,6 +797,7 @@ kind is not (CloudStateOperationKind.Create or CloudStateOperationKind.ContentUp private async ValueTask MarkRescanRequiredAsync(CancellationToken cancellationToken) { + long generation = SignalLoss(); await using ICloudStateTransaction transaction = await _stateStore .BeginTransactionAsync(cancellationToken).ConfigureAwait(false); LocalChangeCheckpoint checkpoint = await ReadCheckpointAsync(transaction, cancellationToken) @@ -809,17 +811,24 @@ await UpsertCheckpointAsync( DateTimeOffset.UtcNow, cancellationToken).ConfigureAwait(false); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + Volatile.Write(ref _persistedLossGeneration, generation); SignalAvailable(); } private async ValueTask PublishAsync(LocalChangeSourceEvent sourceEvent) { + if (sourceEvent.Action is LocalChangeSourceAction.Overflow or LocalChangeSourceAction.Error) + { + SignalLoss(); + } + if (_sourceEvents.Writer.TryWrite(sourceEvent)) { return; } Interlocked.Exchange(ref _overflowSignaled, 1); + SignalLoss(); await ValueTask.CompletedTask; } @@ -834,28 +843,6 @@ private async ValueTask LoadCheckpointAsync(CancellationToken cancellationToken) Volatile.Write(ref _rescanNoticeDelivered, 0); } - private async ValueTask> ListOperationsAsync( - CancellationToken cancellationToken) - { - await using ICloudStateTransaction transaction = await _stateStore - .BeginTransactionAsync(cancellationToken).ConfigureAwait(false); - IReadOnlyList operations = await transaction.Operations - .ListAsync(_options.BatchSize, cancellationToken).ConfigureAwait(false); - await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); - return operations; - } - - private async ValueTask ReadCheckpointAsync( - CancellationToken cancellationToken) - { - await using ICloudStateTransaction transaction = await _stateStore - .BeginTransactionAsync(cancellationToken).ConfigureAwait(false); - LocalChangeCheckpoint checkpoint = await ReadCheckpointAsync(transaction, cancellationToken) - .ConfigureAwait(false); - await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); - return checkpoint; - } - private static async ValueTask ReadCheckpointAsync( ICloudStateTransaction transaction, CancellationToken cancellationToken) diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeScan.cs b/src/CfSharp/LocalChanges/CloudLocalChangeScan.cs index e08f21d..014b3e0 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeScan.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeScan.cs @@ -11,13 +11,19 @@ namespace CfSharp; /// public sealed class CloudLocalChangeScan { - internal CloudLocalChangeScan(Guid storeScope, long throughSequence, bool requiresFullRescan) + internal CloudLocalChangeScan(Guid storeScope, long throughSequence, bool requiresFullRescan, + long lossGeneration, long acknowledgementGeneration) { StoreScope = storeScope; ThroughSequence = throughSequence; RequiresFullRescan = requiresFullRescan; + LossGeneration = lossGeneration; + AcknowledgementGeneration = acknowledgementGeneration; } + internal long LossGeneration { get; } + internal long AcknowledgementGeneration { get; } + /// Gets the opaque identity of this feed's running store-consumer lifetime. /// This value is not a durable database identity or native object binding. public Guid StoreScope { get; } diff --git a/tests/CfSharp.IntegrationTests/RemoteCreationRecoveryTests.cs b/tests/CfSharp.IntegrationTests/RemoteCreationRecoveryTests.cs index 9e79956..a28e365 100644 --- a/tests/CfSharp.IntegrationTests/RemoteCreationRecoveryTests.cs +++ b/tests/CfSharp.IntegrationTests/RemoteCreationRecoveryTests.cs @@ -85,6 +85,12 @@ public async Task NewDirectoryBatchAndFailedCreationCommitReplayWithStableIdenti } Assert.True((await feed.ReadBatchAsync()).RequiresFullRescan); + CloudLocalChangeScan scan = await feed.BeginScanAsync(); + Assert.True(scan.RequiresFullRescan); + CloudLocalChangePage page = await feed.ReadPageAsync(scan, 0, 4); + Assert.True(page.RequiresFullRescan); + Assert.Empty(page.Changes); + Assert.Equal(0, page.LastScannedSequence); await Assert.ThrowsAsync(() => feed.AcknowledgeFullRescanAsync().AsTask()); } diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs index 56b2120..f4cc17d 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs @@ -89,7 +89,7 @@ public async Task OverflowRequiresFullRescanAndCanBeCleared() CloudLocalChangeBatch rescan = await feed.ReadBatchAsync(); Assert.True(rescan.RequiresFullRescan); Assert.Empty(rescan.Changes); - await feed.AcknowledgeFullRescanAsync(); + await AcknowledgePersistedRescanAsync(feed); await source.EmitAsync(new(LocalChangeSourceAction.Created, "after-rescan.txt")); CloudLocalChangeBatch afterRescan = await feed.ReadBatchAsync(); @@ -214,7 +214,7 @@ public async Task InvalidPathAndChannelOverflowRequireAFullRescan() await source.EmitAsync(new(LocalChangeSourceAction.Created, "..\\outside.txt")); CloudLocalChangeBatch invalidPath = await feed.ReadBatchAsync(); Assert.True(invalidPath.RequiresFullRescan); - await feed.AcknowledgeFullRescanAsync(); + await AcknowledgePersistedRescanAsync(feed); await source.EmitAsync(new(LocalChangeSourceAction.Overflow, string.Empty)); CloudLocalChangeBatch overflow = await feed.ReadBatchAsync(); diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs new file mode 100644 index 0000000..1011904 --- /dev/null +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs @@ -0,0 +1,196 @@ +namespace CfSharp.Storage.Sqlite.Tests; + +public sealed partial class CloudLocalChangeFeedTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task LossAfterCaptureBlocksBacklogAndInvalidatesScanAfterAcknowledgement(bool watcherError) + { + await using ICloudStateStore store = await OpenStoreAsync(); + CloudOperationJournalEntry[] rows = await SeedChangesAsync(store, 9); + FakeSource source = new(); + await using CloudLocalChangeFeed feed = CreateFeed(store, source); + await feed.StartAsync(); + CloudLocalChangeScan scan = await feed.BeginScanAsync(); + CloudLocalChangePage first = await feed.ReadPageAsync(scan, 0, 4); + await source.EmitAsync(new(watcherError ? LocalChangeSourceAction.Error : LocalChangeSourceAction.Overflow, + string.Empty, watcherError ? 5 : 0)); + CloudLocalChangePage blocked = await feed.ReadPageAsync(scan, first.LastScannedSequence, 4); + Assert.True(blocked.RequiresFullRescan); + Assert.Empty(blocked.Changes); + Assert.True(blocked.HasMore); + Assert.Equal(first.LastScannedSequence, blocked.LastScannedSequence); + CloudLocalChangeBatch legacy = await feed.ReadBatchAsync(); + Assert.True(legacy.RequiresFullRescan); + Assert.Empty(legacy.Changes); + + await AcknowledgePersistedRescanAsync(feed); + Assert.True((await feed.ReadPageAsync(scan, first.LastScannedSequence, 4)).RequiresFullRescan); + Assert.False((await feed.BeginScanAsync()).RequiresFullRescan); + await using ICloudStateTransaction verify = await store.BeginTransactionAsync(); + Assert.Equal(rows.Select(row => row.OperationId), + (await verify.Operations.ListAsync(20)).Select(row => row.OperationId)); + Assert.False(LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + } + + [Fact] + public async Task LossAtPageTransactionReleaseWithholdsAlreadyReadChanges() + { + await using ICloudStateStore store = await OpenStoreAsync(); + await SeedChangesAsync(store, 2); + RaceStore raceStore = new(store); + FakeSource source = new(); + await using CloudLocalChangeFeed feed = CreateFeed(raceStore, source); + await feed.StartAsync(); + CloudLocalChangeScan scan = await feed.BeginScanAsync(); + raceStore.AfterRollback = () => source.EmitAsync(new(LocalChangeSourceAction.Overflow, string.Empty)); + CloudLocalChangePage page = await feed.ReadPageAsync(scan, 0, 1); + Assert.True(page.RequiresFullRescan); + Assert.Empty(page.Changes); + Assert.Equal(0, page.LastScannedSequence); + Assert.True(page.HasMore); + } + + [Fact] + public async Task LossDuringRescanCommitCannotBeAcknowledgedByThatCommit() + { + await using ICloudStateStore store = await OpenStoreAsync(); + await SeedChangesAsync(store, 2); + RaceStore raceStore = new(store); + FakeSource source = new(); + await using CloudLocalChangeFeed feed = CreateFeed(raceStore, source); + await feed.StartAsync(); + CloudLocalChangeScan original = await feed.BeginScanAsync(); + raceStore.BeforeCommit = () => source.EmitAsync(new(LocalChangeSourceAction.Error, string.Empty, 5)); + await feed.AcknowledgeFullRescanAsync(); + Assert.True((await feed.ReadPageAsync(original, 0, 1)).RequiresFullRescan); + Assert.True((await feed.BeginScanAsync()).RequiresFullRescan); + Assert.True((await feed.ReadBatchAsync()).RequiresFullRescan); + await AcknowledgePersistedRescanAsync(feed); + Assert.False((await feed.BeginScanAsync()).RequiresFullRescan); + } + + [Fact] + public async Task CreationIntentAndRetainedObservationFenceScansUntilExplicitRecovery() + { + await using ICloudStateStore store = await OpenStoreAsync(); + await SeedChangesAsync(store, 2); + await using CloudLocalChangeFeed feed = CreateFeed(store, new FakeSource()); + await feed.StartAsync(); + CloudLocalChangeScan original = await feed.BeginScanAsync(); + string intentName = RemoteCreationIntent.Prefix + "/test"; + string observationName = RemoteCreationIntent.ObservationsPrefix + "/test"; + await using (ICloudStateTransaction write = await store.BeginTransactionAsync()) + { + RemoteCreationIntent intent = new("pending", CloudItemKind.Directory, + new CloudPlaceholderIdentity(Guid.NewGuid(), "pending-remote"), new byte[32], false); + await write.Checkpoints.UpsertAsync(new CloudStateCheckpoint(intentName, intent.Encode(), DateTimeOffset.UtcNow)); + await write.Checkpoints.UpsertAsync(new CloudStateCheckpoint(observationName, [1], DateTimeOffset.UtcNow)); + await write.CommitAsync(); + } + + Assert.True((await feed.BeginScanAsync()).RequiresFullRescan); + Assert.True((await feed.ReadPageAsync(original, 0, 1)).RequiresFullRescan); + await Assert.ThrowsAsync(async () => await feed.AcknowledgeFullRescanAsync()); + await using (ICloudStateTransaction recover = await store.BeginTransactionAsync()) + { + await recover.Checkpoints.RemoveAsync(intentName); + await recover.CommitAsync(); + } + + // Removing a completed creation intent does not implicitly accept its uncertain observations. + Assert.True((await feed.ReadPageAsync(original, 0, 1)).RequiresFullRescan); + await feed.AcknowledgeFullRescanAsync(); + Assert.True((await feed.ReadPageAsync(original, 0, 1)).RequiresFullRescan); + Assert.False((await feed.BeginScanAsync()).RequiresFullRescan); + await using ICloudStateTransaction verify = await store.BeginTransactionAsync(); + Assert.Null(await verify.Checkpoints.GetAsync(observationName)); + Assert.Equal(2, (await verify.Operations.ListAsync(10)).Count); + } + + [Fact] + public async Task DurableRescanFenceSurvivesFullStoreAndFeedRestart() + { + await using (ICloudStateStore store = await OpenStoreAsync()) + await using (CloudLocalChangeFeed feed = CreateFeed(store, new FakeSource())) + { + await SeedChangesAsync(store, 2); + await feed.StartAsync(); + await using ICloudStateTransaction write = await store.BeginTransactionAsync(); + await write.Checkpoints.UpsertAsync(new CloudStateCheckpoint(CloudLocalChangeFeed.CheckpointName, + new LocalChangeCheckpoint(42, true).Encode(), DateTimeOffset.UtcNow)); + await write.CommitAsync(); + } + + await using ICloudStateStore reopened = await OpenStoreAsync(); + await using CloudLocalChangeFeed restarted = CreateFeed(reopened, new FakeSource()); + await restarted.StartAsync(); + CloudLocalChangeScan scan = await restarted.BeginScanAsync(); + Assert.True(scan.RequiresFullRescan); + CloudLocalChangePage page = await restarted.ReadPageAsync(scan, 0, 1); + Assert.Empty(page.Changes); + Assert.True(page.HasMore); + Assert.True(page.RequiresFullRescan); + } + + private static async Task AcknowledgePersistedRescanAsync(CloudLocalChangeFeed feed) + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); + while (true) + { + try + { + await feed.AcknowledgeFullRescanAsync(timeout.Token); + return; + } + catch (InvalidOperationException exception) when (exception.Message.StartsWith("A new loss signal", StringComparison.Ordinal)) + { + await Task.Delay(10, timeout.Token); + } + } + } + + private sealed class RaceStore(ICloudStateStore inner) : ICloudStateStore + { + internal Func? AfterRollback { get; set; } + internal Func? BeforeCommit { get; set; } + public async ValueTask BeginTransactionAsync(CancellationToken cancellationToken = default) => + new RaceTransaction(await inner.BeginTransactionAsync(cancellationToken), this); + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class RaceTransaction(ICloudStateTransaction inner, RaceStore store) : ICloudStateTransaction + { + public ICloudItemStateRepository Items => inner.Items; + public ICloudCheckpointRepository Checkpoints => inner.Checkpoints; + public ICloudOperationJournal Operations => inner.Operations; + public ICloudConflictRepository Conflicts => inner.Conflicts; + public ICloudRemoteBatchRepository RemoteBatches => inner.RemoteBatches; + public ICloudEchoSuppressionRepository EchoSuppressions => inner.EchoSuppressions; + public async ValueTask CommitAsync(CancellationToken cancellationToken = default) + { + Func? hook = store.BeforeCommit; + store.BeforeCommit = null; + if (hook is not null) + { + await hook(); + } + + await inner.CommitAsync(cancellationToken); + } + + public async ValueTask RollbackAsync(CancellationToken cancellationToken = default) + { + await inner.RollbackAsync(cancellationToken); + Func? hook = store.AfterRollback; + store.AfterRollback = null; + if (hook is not null) + { + await hook(); + } + } + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } +} From d7b1b7aa745c1cab72ed7b1e913da4bf391c76df Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:09:10 +0800 Subject: [PATCH 04/40] test(local-changes): cover native backlog scans across roots --- .../CloudLocalChangeScanTests.cs | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 tests/CfSharp.IntegrationTests/CloudLocalChangeScanTests.cs diff --git a/tests/CfSharp.IntegrationTests/CloudLocalChangeScanTests.cs b/tests/CfSharp.IntegrationTests/CloudLocalChangeScanTests.cs new file mode 100644 index 0000000..c127fba --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudLocalChangeScanTests.cs @@ -0,0 +1,195 @@ +using System.Runtime.Versioning; + +using CfSharp.Storage.Sqlite; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed class CloudLocalChangeScanTests +{ + [Fact] + public async Task NativeScansReachActiveSuccessorAndKeepDeferredOperationsAcrossRoots() + { + string area = Path.Combine(Path.GetTempPath(), "CfSharp-native-scans", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(area); + try + { + await using NativeScanRoot first = await NativeScanRoot.OpenAsync(area, "first"); + await using NativeScanRoot second = await NativeScanRoot.OpenAsync(area, "second"); + HashSet deferredPaths = Enumerable.Range(0, 16) + .Select(index => $"Offline\\deferred-{index:D2}.txt") + .ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (string path in deferredPaths) + { + await File.WriteAllTextAsync(Path.Combine(first.RootPath, path), "deferred"); + } + + await WaitForChangesAsync(first.Feed, changes => deferredPaths.All(path => + changes.Any(change => change.RelativePath.Equals(path, StringComparison.OrdinalIgnoreCase)))); + CloudOperationJournalEntry[] deferred; + await using (ICloudStateTransaction capture = await first.Store.BeginTransactionAsync()) + { + deferred = (await capture.Operations.ListAsync(4096)).ToArray(); + } + + Assert.True(deferred.Length >= 16); + await File.WriteAllTextAsync(Path.Combine(first.RootPath, "Active", "live.txt"), "active"); + await File.WriteAllTextAsync(Path.Combine(second.RootPath, "Active", "isolated.txt"), "other root"); + IReadOnlyList all = await WaitForChangesAsync(first.Feed, + changes => changes.Any(change => change.RelativePath == "Active\\live.txt")); + CloudLocalChange accepted = all.First(change => change.RelativePath == "Active\\live.txt"); + Assert.True(accepted.Sequence > deferred[^1].Sequence); + Assert.DoesNotContain(all, change => change.RelativePath == "Active\\isolated.txt"); + // Native notifications can include parent-directory metadata as well as file writes. + // Treat every captured head operation as deferred and retain its original identity. + Assert.Equal(deferred.Take(16).Select(row => row.OperationId), + all.Take(16).Select(change => change.OperationId)); + await first.Feed.AcknowledgeAsync([new CloudLocalChangeAcknowledgement(accepted.OperationId, "accepted-live")]); + + await using (ICloudStateTransaction verify = await first.Store.BeginTransactionAsync()) + { + foreach (CloudOperationJournalEntry original in deferred) + { + CloudOperationJournalEntry? pending = await verify.Operations.GetAsync(original.OperationId); + Assert.NotNull(pending); + Assert.Equal(original.Sequence, pending.Sequence); + Assert.Equal(original.ItemId, pending.ItemId); + Assert.Equal(original.Payload.ToArray(), pending.Payload.ToArray()); + Assert.Equal(original.CreatedAt, pending.CreatedAt); + } + + Assert.Null(await verify.Operations.GetAsync(accepted.OperationId)); + Assert.Equal("accepted-live", (await verify.Items.GetByItemIdAsync(accepted.ItemId!.Value))!.RemoteRevision); + } + + IReadOnlyList isolated = await WaitForChangesAsync(second.Feed, + changes => changes.Any(change => change.RelativePath == "Active\\isolated.txt")); + Assert.DoesNotContain(isolated, change => deferredPaths.Contains(change.RelativePath)); + CloudLocalChangeScan foreign = await first.Feed.BeginScanAsync(); + await Assert.ThrowsAsync(async () => await second.Feed.ReadPageAsync(foreign, 0, 4)); + } + finally + { + Directory.Delete(area, recursive: true); + } + } + + [Fact] + public async Task NativeCreateMoveDeleteDependenciesRemainOrderedAcrossSingleRowPages() + { + string area = Path.Combine(Path.GetTempPath(), "CfSharp-native-scan-order", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(area); + try + { + await using NativeScanRoot root = await NativeScanRoot.OpenAsync(area, "root"); + string before = Path.Combine(root.RootPath, "Active", "before.txt"); + string after = Path.Combine(root.RootPath, "Active", "after.txt"); + await File.WriteAllTextAsync(before, "local"); + IReadOnlyList created = await WaitForChangesAsync(root.Feed, + changes => changes.Any(change => change.Kind == CloudLocalChangeKind.Create)); + Guid itemId = created.First(change => change.Kind == CloudLocalChangeKind.Create).ItemId!.Value; + File.Move(before, after); + await WaitForChangesAsync(root.Feed, changes => changes.Any(change => change.Kind == CloudLocalChangeKind.Move)); + File.Delete(after); + IReadOnlyList changes = await WaitForChangesAsync(root.Feed, + values => values.Any(change => change.Kind == CloudLocalChangeKind.Delete), limit: 1); + CloudLocalChange move = Assert.Single(changes.Where(change => change.Kind == CloudLocalChangeKind.Move)); + CloudLocalChange delete = Assert.Single(changes.Where(change => change.Kind == CloudLocalChangeKind.Delete)); + Assert.Equal("Active\\before.txt", move.PreviousRelativePath); + Assert.Equal("Active\\after.txt", move.RelativePath); + Assert.Equal(move.RelativePath, delete.RelativePath); + Assert.Equal(itemId, move.ItemId); + Assert.Equal(itemId, delete.ItemId); + Assert.True(move.Sequence < delete.Sequence); + Assert.Equal(changes.Select(change => change.Sequence).Order(), changes.Select(change => change.Sequence)); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.True((await verify.Items.GetByItemIdAsync(itemId))!.IsTombstone); + Assert.NotNull(await verify.Operations.GetAsync(move.OperationId)); + Assert.NotNull(await verify.Operations.GetAsync(delete.OperationId)); + } + finally + { + Directory.Delete(area, recursive: true); + } + } + + private static async Task> WaitForChangesAsync(CloudLocalChangeFeed feed, + Func, bool> ready, int limit = 4) + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(20)); + while (true) + { + CloudLocalChangeScan scan = await feed.BeginScanAsync(timeout.Token); + List changes = []; + long cursor = 0; + CloudLocalChangePage page; + do + { + page = await feed.ReadPageAsync(scan, cursor, limit, timeout.Token); + Assert.False(page.RequiresFullRescan); + Assert.InRange(page.Changes.Count, 0, limit); + changes.AddRange(page.Changes); + cursor = page.LastScannedSequence; + } + while (page.HasMore); + + if (ready(changes)) + { + return changes; + } + + await Task.Delay(20, timeout.Token); + } + } + + private sealed class NativeScanRoot(CloudFileSystem fileSystem, CapturingFactory factory) : IAsyncDisposable + { + internal string RootPath => fileSystem.SyncRootPath; + internal ICloudStateStore Store => factory.Store!; + internal CloudLocalChangeFeed Feed { get; } = fileSystem.CreateLocalChangeFeed( + new CloudLocalChangeFeedOptions { BatchSize = 4, BufferCapacity = 1024 }); + + internal static async Task OpenAsync(string area, string name) + { + string rootPath = Path.Combine(area, name); + Directory.CreateDirectory(Path.Combine(rootPath, "Offline")); + Directory.CreateDirectory(Path.Combine(rootPath, "Active")); + CapturingFactory factory = new(new SqliteCloudStateStoreFactory(Path.Combine(area, name + ".db"))); + CloudFileSystem fileSystem = CloudFileSystem.CreateBuilder(rootPath).WithStateStore(factory) + .WithRegistration(SyncRootRegistrationOptions.CreateBuilder("CfSharp Native Scan", "1.0-test") + .WithProviderId(Guid.NewGuid()).WithRootMarkedInSync().Build()).Build(); + bool registered = false; + try + { + await fileSystem.StartAsync(); + registered = true; + NativeScanRoot root = new(fileSystem, factory); + await root.Feed.StartAsync(); + return root; + } + catch + { + await fileSystem.DisposeAsync(); + if (registered) + { + CloudSyncRoot.Open(rootPath).Unregister(); + } + + throw; + } + } + + public async ValueTask DisposeAsync() + { + await fileSystem.DisposeAsync(); + CloudSyncRoot.Open(RootPath).Unregister(); + } + } + + private sealed class CapturingFactory(ICloudStateStoreFactory inner) : ICloudStateStoreFactory + { + internal ICloudStateStore? Store { get; private set; } + public async ValueTask OpenAsync(CloudStateStoreContext context, + CancellationToken cancellationToken = default) => Store = await inner.OpenAsync(context, cancellationToken); + } +} From 7c350a40ed2788d47b2aa790af04c86a17333532 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:13:08 +0800 Subject: [PATCH 05/40] refactor(items): share transactional subtree path projection --- .../CloudFileSystem.NamespaceOperations.cs | 25 +------- .../CloudDirectoryStateProjection.cs | 38 +++++++++++ .../CloudDirectoryStateProjectionTests.cs | 64 +++++++++++++++++++ 3 files changed, 104 insertions(+), 23 deletions(-) create mode 100644 src/CfSharp/Persistence/CloudDirectoryStateProjection.cs create mode 100644 tests/CfSharp.Tests/Persistence/CloudDirectoryStateProjectionTests.cs diff --git a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs index 125464b..d4b3a54 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs @@ -294,29 +294,8 @@ private static async ValueTask PersistMovedStateAsync( await using ICloudStateTransaction transaction = await stateStore .BeginTransactionAsync(cancellationToken) .ConfigureAwait(false); - DateTimeOffset updatedAt = DateTimeOffset.UtcNow; - foreach (CloudItemState item in sourceEntries) - { - string suffix = item.RelativePath.Length == sourceRelativePath.Length - ? string.Empty - : item.RelativePath[sourceRelativePath.Length..] - .TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); - string updatedPath = suffix.Length == 0 - ? destinationRelativePath - : Path.Combine(destinationRelativePath, suffix); - await transaction.Items.UpsertAsync( - new CloudItemState( - item.ItemId, - item.RemoteId, - updatedPath, - item.Kind, - item.RemoteRevision, - item.LocalFileId, - item.IsTombstone, - updatedAt), - cancellationToken).ConfigureAwait(false); - } - + await CloudDirectoryStateProjection.ProjectAsync(transaction, sourceEntries, + sourceRelativePath, destinationRelativePath, DateTimeOffset.UtcNow, cancellationToken).ConfigureAwait(false); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); } diff --git a/src/CfSharp/Persistence/CloudDirectoryStateProjection.cs b/src/CfSharp/Persistence/CloudDirectoryStateProjection.cs new file mode 100644 index 0000000..1861ccc --- /dev/null +++ b/src/CfSharp/Persistence/CloudDirectoryStateProjection.cs @@ -0,0 +1,38 @@ +namespace CfSharp; + +// The caller owns the transaction and namespace-operation lifetime. This helper never commits, +// performs native I/O, acknowledges journal rows, or infers native ownership from durable paths. +internal static class CloudDirectoryStateProjection +{ + internal static async ValueTask ProjectAsync(ICloudStateTransaction transaction, + IReadOnlyList sourceEntries, string sourceRelativePath, + string destinationRelativePath, DateTimeOffset updatedAt, CancellationToken cancellationToken) + { + // Compute every path before writing any row. Segment validation prevents a similarly + // named sibling (Docs2) from being treated as a descendant of Docs. + (CloudItemState Item, string Path)[] entries = sourceEntries.Select(item => + (item, MapPath(item.RelativePath, sourceRelativePath, destinationRelativePath))).ToArray(); + foreach ((CloudItemState item, string path) in entries) + { + await transaction.Items.UpsertAsync(new CloudItemState(item.ItemId, item.RemoteId, path, + item.Kind, item.RemoteRevision, item.LocalFileId, item.IsTombstone, updatedAt), + cancellationToken).ConfigureAwait(false); + } + } + + internal static string MapPath(string relativePath, string sourceRelativePath, string destinationRelativePath) + { + string normalizedPath = relativePath.Replace('/', '\\'); + string normalizedSource = sourceRelativePath.Replace('/', '\\'); + if (!string.Equals(normalizedPath, normalizedSource, StringComparison.OrdinalIgnoreCase) && + !normalizedPath.StartsWith(normalizedSource + "\\", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException("The durable item is outside the source subtree.", nameof(relativePath)); + } + + string suffix = relativePath.Length == sourceRelativePath.Length + ? string.Empty + : relativePath[sourceRelativePath.Length..].TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + return suffix.Length == 0 ? destinationRelativePath : Path.Combine(destinationRelativePath, suffix); + } +} diff --git a/tests/CfSharp.Tests/Persistence/CloudDirectoryStateProjectionTests.cs b/tests/CfSharp.Tests/Persistence/CloudDirectoryStateProjectionTests.cs new file mode 100644 index 0000000..798107b --- /dev/null +++ b/tests/CfSharp.Tests/Persistence/CloudDirectoryStateProjectionTests.cs @@ -0,0 +1,64 @@ +namespace CfSharp.Tests.Persistence; + +public sealed class CloudDirectoryStateProjectionTests +{ + [Fact] + public async Task ProjectionRetainsFieldsAndJournalFactsAndObeysCallerRollback() + { + await using ICloudStateStore store = await InMemoryCloudStateStoreContractTests.CreateFactoryForTesting() + .OpenAsync(new CloudStateStoreContext(Path.GetTempPath())); + CloudItemState directory = new(Guid.NewGuid(), "dir", "Docs", CloudItemKind.Directory, + "directory-revision", 11, false, DateTimeOffset.UtcNow); + CloudItemState child = new(Guid.NewGuid(), "child", "Docs\\Deep\\child.txt", CloudItemKind.File, + "child-revision", 12, true, DateTimeOffset.UtcNow); + CloudItemState sibling = new(Guid.NewGuid(), "sibling", "Docs2\\sibling.txt", CloudItemKind.File, + "sibling-revision", 13, false, DateTimeOffset.UtcNow); + CloudOperationJournalEntry operation; + await using (ICloudStateTransaction seed = await store.BeginTransactionAsync()) + { + foreach (CloudItemState item in new[] { directory, child, sibling }) + { + await seed.Items.UpsertAsync(item); + } + + operation = await seed.Operations.EnqueueAsync(new CloudOperationJournalEntry( + Guid.NewGuid(), CloudStateOperationKind.Move, directory.ItemId, [1, 2, 3], DateTimeOffset.UtcNow)); + await seed.CommitAsync(); + } + + DateTimeOffset updatedAt = DateTimeOffset.UtcNow.AddSeconds(1); + await using (ICloudStateTransaction rollback = await store.BeginTransactionAsync()) + { + await CloudDirectoryStateProjection.ProjectAsync(rollback, [directory, child], "Docs", "Moved", updatedAt, default); + } + + await using (ICloudStateTransaction project = await store.BeginTransactionAsync()) + { + Assert.Equal("Docs", (await project.Items.GetByItemIdAsync(directory.ItemId))!.RelativePath); + await CloudDirectoryStateProjection.ProjectAsync(project, [directory, child], "Docs", "Moved", updatedAt, default); + await project.CommitAsync(); + } + + await using ICloudStateTransaction verify = await store.BeginTransactionAsync(); + CloudItemState movedChild = (await verify.Items.GetByItemIdAsync(child.ItemId))!; + Assert.Equal("Moved\\Deep\\child.txt", movedChild.RelativePath); + Assert.Equal(child.RemoteId, movedChild.RemoteId); + Assert.Equal(child.RemoteRevision, movedChild.RemoteRevision); + Assert.Equal(child.LocalFileId, movedChild.LocalFileId); + Assert.True(movedChild.IsTombstone); + Assert.Equal(updatedAt, movedChild.UpdatedAt); + Assert.Equal(sibling.RelativePath, (await verify.Items.GetByItemIdAsync(sibling.ItemId))!.RelativePath); + CloudOperationJournalEntry retained = (await verify.Operations.GetAsync(operation.OperationId))!; + Assert.Equal(operation.Sequence, retained.Sequence); + Assert.Equal(operation.Payload.ToArray(), retained.Payload.ToArray()); + Assert.Equal(operation.CreatedAt, retained.CreatedAt); + } + + [Fact] + public void MappingHonorsSegmentBoundariesAndCaseOnlyRenames() + { + Assert.Equal("docs", CloudDirectoryStateProjection.MapPath("Docs", "Docs", "docs")); + Assert.Equal("Moved\\a\\b.txt", CloudDirectoryStateProjection.MapPath("DOCS\\a\\b.txt", "Docs", "Moved")); + Assert.Throws(() => CloudDirectoryStateProjection.MapPath("Docs2\\a.txt", "Docs", "Moved")); + } +} From 74b0e9da728774ed43fda7d5e8eac761d8b7e099 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:34:10 +0800 Subject: [PATCH 06/40] feat(items): expose complete native bindings for directories --- docs/placeholders.md | 12 ++++ .../Internal/WindowsFileMetadata.cs | 50 +++++++++++++++++ src/CfSharp/Items/CloudItemInspector.cs | 2 +- src/CfSharp/Items/CloudItemSnapshot.cs | 7 ++- src/CfSharp/Items/CloudLocalFileBinding.cs | 8 +-- .../CloudItemNamespaceOperationTests.cs | 3 + .../WindowsFileMetadataTests.cs | 56 +++++++++++++++++++ tests/CfSharp.Tests/CloudItemTests.cs | 22 +++++++- 8 files changed, 151 insertions(+), 9 deletions(-) create mode 100644 tests/CfSharp.Native.Tests/WindowsFileMetadataTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index 9fe4ddd..4f71793 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -3,6 +3,18 @@ Placeholder operations use immutable, kind-specific specifications rather than exposing native unions and flag combinations directly to application code. +## Observe native object bindings + +`CloudItemSnapshot.LocalBinding` exposes the complete volume serial number, sync-root directory +ID, and local object ID for files and directories on capable Windows storage. IDs retain all native +bits and are opaque comparison values. Inspection reads metadata without reading or hydrating +content and retains no handle or protection lifetime. Unsupported reparse targets, missing objects, +and storage without complete IDs return null. + +A rename preserves the binding; a replacement can have a different binding even when its path, +remote ID, or placeholder identity matches. A current target observation alone cannot prove that +it belonged to an earlier source. File content confirmation still requires its accepted upload proof. + ## Create a placeholder ```csharp diff --git a/src/CfSharp.Native/Internal/WindowsFileMetadata.cs b/src/CfSharp.Native/Internal/WindowsFileMetadata.cs index 75f98d9..1bb87ad 100644 --- a/src/CfSharp.Native/Internal/WindowsFileMetadata.cs +++ b/src/CfSharp.Native/Internal/WindowsFileMetadata.cs @@ -32,6 +32,52 @@ internal static FileIdentity ReadIdentity(nint handle) return value; } + internal static string ReadFinalPath(nint handle) + { + char[] buffer = new char[260]; + while (true) + { + uint length; + fixed (char* pointer = buffer) + { + // Normalized DOS spelling observes the current namespace through the existing + // object handle, including a case-only rename. Successful lengths exclude NUL; + // insufficient-buffer lengths include it. Never close a borrowed handle here. + // https://learn.microsoft.com/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlew + length = GetFinalPathNameByHandle(handle, pointer, (uint)buffer.Length, 0); + } + + if (length == 0) + { + throw new Win32Exception(Marshal.GetLastPInvokeError()); + } + + if (length < buffer.Length) + { + string path = new(buffer, 0, (int)length); + if (path.StartsWith(@"\\?\UNC\", StringComparison.OrdinalIgnoreCase)) + { + path = @"\\" + path[8..]; + } + else if (path.StartsWith(@"\\?\", StringComparison.Ordinal)) + { + path = path[4..]; + } + + return Path.TrimEndingDirectorySeparator(Path.GetFullPath(path)); + } + + // Bound repeated growth even if a concurrent external rename changes the required + // length between calls. This query never follows a final link opened as reparse data. + if (length > 32768 || buffer.Length == 32768) + { + throw new PathTooLongException("The native namespace path exceeds the Windows path limit."); + } + + buffer = new char[Math.Min(32768, checked((int)length + 1))]; + } + } + [SupportedOSPlatform("windows10.0.16299")] internal static NativeFileMetadata Read(nint handle, nint protectedHandle = 0) { @@ -122,6 +168,10 @@ private static partial SafeFileHandle CreateFile(string path, uint access, uint [UnmanagedCallConv(CallConvs = new[] { typeof(CallConvStdcall) })] [return: MarshalAs(UnmanagedType.Bool)] private static partial bool GetFileInformationByHandleEx(nint handle, int informationClass, void* value, uint size); + + [LibraryImport("kernel32.dll", EntryPoint = "GetFinalPathNameByHandleW", SetLastError = true)] + [UnmanagedCallConv(CallConvs = new[] { typeof(CallConvStdcall) })] + private static partial uint GetFinalPathNameByHandle(nint handle, char* path, uint capacity, uint flags); } internal sealed record NativeFileMetadata(WindowsFileMetadata.FileIdentity Identity, long Length, diff --git a/src/CfSharp/Items/CloudItemInspector.cs b/src/CfSharp/Items/CloudItemInspector.cs index edc9b4f..5ebecb6 100644 --- a/src/CfSharp/Items/CloudItemInspector.cs +++ b/src/CfSharp/Items/CloudItemInspector.cs @@ -170,7 +170,7 @@ internal static unsafe LocalCloudItemInspection Inspect( : CloudSynchronizationState.NotApplicable), LocalFileId = placeholder?.FileId, SyncRootFileId = placeholder?.SyncRootFileId, - LocalBinding = actualKind == CloudItemKind.File && syncRootPath is not null && + LocalBinding = syncRootPath is not null && (!attributes.HasFlag(FileAttributes.ReparsePoint) || placeholder is not null) ? CloudLocalFileBindingPlatform.TryRead(handle.DangerousGetHandle(), syncRootPath) : null, diff --git a/src/CfSharp/Items/CloudItemSnapshot.cs b/src/CfSharp/Items/CloudItemSnapshot.cs index e62804e..fabf5cb 100644 --- a/src/CfSharp/Items/CloudItemSnapshot.cs +++ b/src/CfSharp/Items/CloudItemSnapshot.cs @@ -108,10 +108,11 @@ internal CloudItemSnapshot( /// Gets the complete native volume, root, and file binding when available. /// - /// Available for ordinary files as well as Cloud Files placeholders. No content is read or + /// Available for ordinary files and directories as well as Cloud Files placeholders. No content is read or /// hydrated. This observation holds no lock and is not a USN or content proof. Null denotes - /// absence, directories, or unsupported reparse targets. Retain the upload-time value rather - /// than replacing it with a later observation when confirming accepted content. + /// absence, storage without complete IDs, or unsupported reparse targets. Retain the upload-time + /// value when confirming accepted content, or capture a directory move proof before renaming; + /// a later observation alone cannot prove historical ownership. /// public CloudLocalFileBinding? LocalBinding { get; } diff --git a/src/CfSharp/Items/CloudLocalFileBinding.cs b/src/CfSharp/Items/CloudLocalFileBinding.cs index 17463a1..53849dc 100644 --- a/src/CfSharp/Items/CloudLocalFileBinding.cs +++ b/src/CfSharp/Items/CloudLocalFileBinding.cs @@ -1,10 +1,10 @@ namespace CfSharp; -/// Identifies a local file object within one actual sync root and volume. +/// Identifies a local file or directory object within one actual sync root and volume. /// /// Immutable, serializable comparison data with no handle ownership or protection lifetime. /// File IDs preserve all 128 native bits; their Guid representation is opaque, not a UUID. -/// Capture this value before uploading and retain it with the accepted content proof. A later +/// Capture this value before uploading or moving a directory and retain the corresponding proof. A later /// observation of a replacement cannot substitute for that proof. IDs can be reused after /// deletion; this value does not establish remote identity, content equality, or a lock. /// Supported on Windows 10 version 1709 and later with file-ID-capable storage. @@ -14,7 +14,7 @@ public sealed record CloudLocalFileBinding /// Initializes an immutable native object binding. /// Native volume serial number. /// Complete native ID of the owning root directory. - /// Complete native ID of the file. + /// Complete native ID of the file or directory object. /// An identifier is empty. public CloudLocalFileBinding(ulong volumeSerialNumber, Guid syncRootFileId, Guid localFileId) { @@ -34,6 +34,6 @@ public CloudLocalFileBinding(ulong volumeSerialNumber, Guid syncRootFileId, Guid /// Gets the opaque 128-bit native ID of the actual root directory. public Guid SyncRootFileId { get; } - /// Gets the opaque 128-bit native ID of the file object. + /// Gets the opaque 128-bit native ID of the file or directory object. public Guid LocalFileId { get; } } diff --git a/tests/CfSharp.IntegrationTests/CloudItemNamespaceOperationTests.cs b/tests/CfSharp.IntegrationTests/CloudItemNamespaceOperationTests.cs index d3c12b7..67e755a 100644 --- a/tests/CfSharp.IntegrationTests/CloudItemNamespaceOperationTests.cs +++ b/tests/CfSharp.IntegrationTests/CloudItemNamespaceOperationTests.cs @@ -66,6 +66,8 @@ await fileSystem.Root.CreatePlaceholdersAsync( [sourceSpec, destinationSpec, renameSpec, replaceSpec, emptySpec]); CloudDirectory source = fileSystem.GetDirectory("Source"); + CloudLocalFileBinding sourceBinding = Assert.IsType( + (await source.InspectAsync()).LocalBinding); CloudDirectory destination = fileSystem.GetDirectory("Destination"); CloudFilePlaceholderSpec sourceFileSpec = FileSpec( "source-file.bin", @@ -113,6 +115,7 @@ await fileSystem.Root.CreatePlaceholdersAsync( CloudDirectory movedDirectory = Assert.IsType(movedDirectoryResult.Item); Assert.Equal(4, movedDirectoryResult.DurableStateEntriesUpdated); Assert.Equal(sourceSpec.Identity.ItemId, movedDirectoryResult.Snapshot.ItemId); + Assert.Equal(sourceBinding, movedDirectoryResult.Snapshot.LocalBinding); CloudFile movedDeepFile = movedDirectory.GetFile(Path.Combine("Nested", "deep.bin")); Assert.Equal(deepFileSpec.Identity.ItemId, (await movedDeepFile.InspectAsync()).ItemId); Assert.False((await source.GetFile(Path.Combine("Nested", "deep.bin")).InspectAsync()).Exists); diff --git a/tests/CfSharp.Native.Tests/WindowsFileMetadataTests.cs b/tests/CfSharp.Native.Tests/WindowsFileMetadataTests.cs new file mode 100644 index 0000000..6a53a6d --- /dev/null +++ b/tests/CfSharp.Native.Tests/WindowsFileMetadataTests.cs @@ -0,0 +1,56 @@ +using System.ComponentModel; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace CfSharp.Native.Tests; + +public sealed class WindowsFileMetadataTests +{ + [Fact] + public void FinalPathImportHasExplicitUtf16PointerAndWindowsCallingConvention() + { + MethodInfo method = typeof(WindowsFileMetadata).GetMethod("GetFinalPathNameByHandle", + BindingFlags.NonPublic | BindingFlags.Static)!; + LibraryImportAttribute import = method.GetCustomAttribute()!; + Assert.Equal("kernel32.dll", import.LibraryName); + Assert.Equal("GetFinalPathNameByHandleW", import.EntryPoint); + Assert.True(import.SetLastError); + Assert.Equal(typeof(uint), method.ReturnType); + Assert.Equal(new[] { typeof(nint), typeof(char).MakePointerType(), typeof(uint), typeof(uint) }, + method.GetParameters().Select(parameter => parameter.ParameterType)); + Assert.Equal(new[] { typeof(CallConvStdcall) }, method.GetCustomAttribute()!.CallConvs); + } + + [Fact] + public void FinalPathObservesLongDirectoryRenameAndRetainsBorrowedHandleIdentity() + { + string area = Path.Combine(Path.GetTempPath(), "CfSharp-final-path", Guid.NewGuid().ToString("N")); + string parent = Path.Combine(area, new string('a', 64), new string('b', 64), new string('c', 64)); + string source = Path.Combine(parent, "OriginalCase"); + string destination = Path.Combine(parent, "originalcase"); + Directory.CreateDirectory(source); + try + { + using var handle = WindowsFileMetadata.Open(@"\\?\" + source); + WindowsFileMetadata.FileIdentity identity = WindowsFileMetadata.ReadIdentity(handle.DangerousGetHandle()); + Assert.True(source.Length > 260); + Assert.Equal(source, WindowsFileMetadata.ReadFinalPath(handle.DangerousGetHandle())); + Directory.Move(@"\\?\" + source, @"\\?\" + destination); + Assert.Equal(destination, WindowsFileMetadata.ReadFinalPath(handle.DangerousGetHandle())); + Assert.Equal(identity.FileId, WindowsFileMetadata.ReadIdentity(handle.DangerousGetHandle()).FileId); + Assert.False(handle.IsClosed); + } + finally + { + Directory.Delete(area, recursive: true); + } + } + + [Fact] + public void FinalPathPreservesNativeFailure() + { + Win32Exception failure = Assert.Throws(() => WindowsFileMetadata.ReadFinalPath(0)); + Assert.Equal(6, failure.NativeErrorCode); + } +} diff --git a/tests/CfSharp.Tests/CloudItemTests.cs b/tests/CfSharp.Tests/CloudItemTests.cs index 26adafc..10f1c73 100644 --- a/tests/CfSharp.Tests/CloudItemTests.cs +++ b/tests/CfSharp.Tests/CloudItemTests.cs @@ -26,7 +26,9 @@ public async Task InspectionReturnsFreshImmutableFileAndRootSnapshots() Assert.Equal(3, first.Length); Assert.NotNull(first.LocalBinding); Assert.Equal(first.LocalBinding, second.LocalBinding); - Assert.Null(rootSnapshot.LocalBinding); + Assert.NotNull(rootSnapshot.LocalBinding); + Assert.Equal(rootSnapshot.LocalBinding.LocalFileId, rootSnapshot.LocalBinding.SyncRootFileId); + Assert.Equal(rootSnapshot.LocalBinding.SyncRootFileId, first.LocalBinding.SyncRootFileId); Assert.Equal(12, second.Length); Assert.Equal(3, first.Length); Assert.True(first.Exists); @@ -38,6 +40,23 @@ public async Task InspectionReturnsFreshImmutableFileAndRootSnapshots() Assert.Null(fileSystem.Root.Parent); } + [Fact] + public async Task DirectoryBindingSurvivesRenameAndDistinguishesReplacement() + { + using TestDirectory root = new(); + string sourcePath = Path.Combine(root.Path, "Source"); + string destinationPath = Path.Combine(root.Path, "Moved"); + Directory.CreateDirectory(sourcePath); + await using CloudFileSystem fileSystem = await StartAsync(root.Path, new InspectionStore()); + CloudLocalFileBinding before = Assert.IsType( + (await fileSystem.GetDirectory("Source").InspectAsync()).LocalBinding); + Directory.Move(sourcePath, destinationPath); + Assert.Equal(before, (await fileSystem.GetDirectory("Moved").InspectAsync()).LocalBinding); + Assert.Null((await fileSystem.GetDirectory("Source").InspectAsync()).LocalBinding); + Directory.CreateDirectory(sourcePath); + Assert.NotEqual(before, (await fileSystem.GetDirectory("Source").InspectAsync()).LocalBinding); + } + [Fact] public async Task MissingLocalItemStillReturnsDurableTombstoneState() { @@ -263,6 +282,7 @@ public async Task EnumerationIsFreshAndDoesNotFollowDirectoryLinks() IReadOnlyList first = await CollectAsync( fileSystem.Root.EnumerateLocalChildrenAsync(recursive)); + Assert.Null((await fileSystem.GetDirectory("alias").InspectAsync()).LocalBinding); await File.WriteAllTextAsync(Path.Combine(root.Path, "added.txt"), "new"); IReadOnlyList second = await CollectAsync( fileSystem.Root.EnumerateLocalChildrenAsync(recursive)); From 82b483a73ea11331e33e88aa313b9fdbb3bda1bb Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:43:40 +0800 Subject: [PATCH 07/40] fix(sqlite): fence durable directory recovery metadata --- docs/state-store-sqlite.md | 16 ++- .../SqliteCloudStateStoreFactory.cs | 13 ++- .../SqliteCloudStateStoreTests.cs | 99 +++++++++++++++++-- 3 files changed, 117 insertions(+), 11 deletions(-) diff --git a/docs/state-store-sqlite.md b/docs/state-store-sqlite.md index 2566842..9cc2ba5 100644 --- a/docs/state-store-sqlite.md +++ b/docs/state-store-sqlite.md @@ -49,6 +49,16 @@ Pages own no reader or transaction and do not acknowledge operations or update i The legacy `ICloudOperationJournal` contract is unchanged: custom stores may opt into paging on their repository, preserving monotonic enqueue sequences and transactional read semantics. -### Schema 5 recovery fence - -Schema 5 preserves the existing table layout and data while fencing the remote placeholder creation and observation-reconciliation protocol. Versions 0 through 4 upgrade in place. Older libraries reject the newer version rather than ignore pending creation records. Do not downgrade the schema number manually. Back up state before upgrading and use matching core and SQLite packages. +### Schema 6 recovery fence + +Schema 6 preserves the existing table layout and data while fencing directory object provenance, +move preparation, and completion-receipt recovery. It includes schema 5's remote placeholder +creation and observation-reconciliation fence. Versions 0 through 5 upgrade in place, retaining +item IDs, revisions, tombstones, journal sequences and payloads, suppressions, and checkpoints. +Older SQLite packages reject version 6 before accessing its recovery metadata. Do not downgrade +the schema number manually. Back up state before upgrading and use matching core and SQLite packages. + +Custom stores must enforce the same protocol boundary when opening persistent state with older +runtimes. Preserve CfSharp-owned checkpoint names and values transactionally, including immutable +preparations and receipts; never treat them as disposable caches or infer native ownership from an +item path. The schema fence changes protocol compatibility, not the replaceable core interfaces. diff --git a/src/CfSharp.Storage.Sqlite/SqliteCloudStateStoreFactory.cs b/src/CfSharp.Storage.Sqlite/SqliteCloudStateStoreFactory.cs index 98bcf45..555adb2 100644 --- a/src/CfSharp.Storage.Sqlite/SqliteCloudStateStoreFactory.cs +++ b/src/CfSharp.Storage.Sqlite/SqliteCloudStateStoreFactory.cs @@ -777,7 +777,7 @@ private struct FileAttributeTagInfo internal static class SqliteSchema { - internal const int CurrentVersion = 5; + internal const int CurrentVersion = 6; internal const string PathCollationName = "CFSHARP_UNICODE_NOCASE"; internal static async Task InitializeAsync( @@ -868,6 +868,17 @@ await MigrateVersionFourAsync(connection, cancellationToken) await upgrade.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } + if (version < 6) + { + // Directory object provenance, immutable move preparations, and completion + // receipts use checkpoints, but change the recovery protocol. The version fence + // prevents older runtimes from silently performing unverified namespace replay. + // No table, existing checkpoint, item field, or journal sequence is rewritten. + await using SqliteCommand upgrade = connection.CreateCommand(); + upgrade.CommandText = "UPDATE cfsharp_schema SET version = 6 WHERE singleton = 1;"; + await upgrade.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + await ValidateCurrentVersionAsync( connection, databasePath, diff --git a/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs index 211a482..3565844 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs @@ -375,7 +375,7 @@ version INTEGER NOT NULL } Assert.Equal( - 5L, + 6L, await ExecuteScalarInt64Async( _databasePath, "SELECT version FROM cfsharp_schema WHERE singleton = 1;")); @@ -387,7 +387,7 @@ await ExecuteScalarInt64Async( } [Fact] - public async Task VersionOneRemoteBatchSchemaMigratesToVersionFive() + public async Task VersionOneRemoteBatchSchemaMigratesToVersionSix() { await using (ICloudStateStore store = await CreateFactory().OpenAsync(CreateContext())) { @@ -402,7 +402,7 @@ await ExecuteSqlAsync( } Assert.Equal( - 5L, + 6L, await ExecuteScalarInt64Async( _databasePath, "SELECT version FROM cfsharp_schema WHERE singleton = 1;")); @@ -415,7 +415,7 @@ await ExecuteScalarInt64Async( } [Fact] - public async Task VersionTwoEchoSuppressionSchemaMigratesToVersionFive() + public async Task VersionTwoEchoSuppressionSchemaMigratesToVersionSix() { await using (ICloudStateStore store = await CreateFactory().OpenAsync(CreateContext())) { @@ -430,7 +430,7 @@ await ExecuteSqlAsync( } Assert.Equal( - 5L, + 6L, await ExecuteScalarInt64Async( _databasePath, "SELECT version FROM cfsharp_schema WHERE singleton = 1;")); @@ -459,10 +459,95 @@ public async Task VersionFourUpgradePreservesRecoveryMetadata() Assert.Equal(new byte[] { 7 }, (await transaction.Checkpoints.GetAsync("existing/checkpoint"))!.Value.ToArray()); } - Assert.Equal(5L, await ExecuteScalarInt64Async(_databasePath, + Assert.Equal(6L, await ExecuteScalarInt64Async(_databasePath, "SELECT version FROM cfsharp_schema WHERE singleton = 1;")); } + [Fact] + public async Task VersionFiveUpgradePreservesCoordinationRowsAndSequenceAllocation() + { + Guid itemId = Guid.NewGuid(); + Guid operationId = Guid.NewGuid(); + Guid conflictId = Guid.NewGuid(); + Guid suppressionId = Guid.NewGuid(); + DateTimeOffset timestamp = DateTimeOffset.UtcNow; + CloudOperationJournalEntry original; + await using (ICloudStateStore store = await CreateFactory().OpenAsync(CreateContext())) + await using (ICloudStateTransaction write = await store.BeginTransactionAsync()) + { + await write.Items.UpsertAsync(new CloudItemState(itemId, "retained-remote", "Docs\\gone.txt", + CloudItemKind.File, "retained-revision", 321, true, timestamp)); + await write.Checkpoints.UpsertAsync(new CloudStateCheckpoint("retained/checkpoint", [1, 2], timestamp)); + original = await write.Operations.EnqueueAsync(new CloudOperationJournalEntry(operationId, + CloudStateOperationKind.Move, itemId, [3, 4], timestamp, 3, timestamp.AddMinutes(1))); + await write.Conflicts.UpsertAsync(new CloudConflictState(conflictId, itemId, CloudStateConflictKind.Move, [5, 6], timestamp)); + await write.RemoteBatches.UpsertAsync(new CloudRemoteBatchState("retained-batch", [7], 1, 2, + CloudRemoteBatchStatus.Applying, [8], timestamp, new byte[] { 9 }, "retained-change")); + await write.EchoSuppressions.UpsertAsync(new CloudEchoSuppressionState(suppressionId, + itemId, CloudStateOperationKind.Move, "Docs\\gone.txt", [10], timestamp.AddMinutes(5), "old.txt", 2)); + await write.CommitAsync(); + } + + await ExecuteSqlAsync(_databasePath, "UPDATE cfsharp_schema SET version = 5 WHERE singleton = 1;"); + await using (ICloudStateStore reopened = await CreateFactory().OpenAsync(CreateContext())) + await using (ICloudStateTransaction verify = await reopened.BeginTransactionAsync()) + { + CloudItemState item = (await verify.Items.GetByItemIdAsync(itemId))!; + Assert.Equal("Docs\\gone.txt", item.RelativePath); + Assert.Equal("retained-remote", item.RemoteId); + Assert.Equal("retained-revision", item.RemoteRevision); + Assert.Equal(321, item.LocalFileId); + Assert.True(item.IsTombstone); + Assert.Equal(timestamp, item.UpdatedAt); + Assert.Equal(new byte[] { 1, 2 }, (await verify.Checkpoints.GetAsync("retained/checkpoint"))!.Value.ToArray()); + CloudOperationJournalEntry operation = (await verify.Operations.GetAsync(operationId))!; + Assert.Equal(original.Sequence, operation.Sequence); + Assert.Equal(itemId, operation.ItemId); + Assert.Equal(new byte[] { 3, 4 }, operation.Payload.ToArray()); + Assert.Equal(timestamp, operation.CreatedAt); + Assert.Equal(3, operation.AttemptCount); + Assert.Equal(original.RetryAfter, operation.RetryAfter); + Assert.Equal(new byte[] { 5, 6 }, (await verify.Conflicts.GetAsync(conflictId))!.Payload.ToArray()); + CloudRemoteBatchState batch = (await verify.RemoteBatches.GetAsync("retained-batch"))!; + Assert.Equal(new byte[] { 7 }, batch.Cursor.ToArray()); + Assert.Equal(new byte[] { 9 }, batch.Fingerprint.ToArray()); + Assert.Equal("retained-change", batch.LastAppliedChangeId); + CloudEchoSuppressionState suppression = (await verify.EchoSuppressions.GetAsync(suppressionId))!; + Assert.Equal("old.txt", suppression.PreviousRelativePath); + Assert.Equal(2, suppression.RemainingObservations); + Assert.Equal(new byte[] { 10 }, suppression.Payload.ToArray()); + CloudOperationJournalEntry next = await verify.Operations.EnqueueAsync(new CloudOperationJournalEntry( + Guid.NewGuid(), CloudStateOperationKind.MetadataUpdate, itemId, [11], timestamp)); + Assert.True(next.Sequence > original.Sequence); + await verify.CommitAsync(); + } + + Assert.Equal(6L, await ExecuteScalarInt64Async(_databasePath, + "SELECT version FROM cfsharp_schema WHERE singleton = 1;")); + } + + [Fact] + public async Task VersionSixFenceSurvivesProcessExitBeforeApplicationCommit() + { + await using (ICloudStateStore store = await CreateFactory().OpenAsync(CreateContext())) + await using (ICloudStateTransaction seed = await store.BeginTransactionAsync()) + { + await seed.Checkpoints.UpsertAsync(new CloudStateCheckpoint("retained/before-crash", [12], DateTimeOffset.UtcNow)); + await seed.CommitAsync(); + } + + await ExecuteSqlAsync(_databasePath, "UPDATE cfsharp_schema SET version = 5 WHERE singleton = 1;"); + (int exitCode, _) = await RunCrashHarnessAsync("before-commit", _databasePath, _syncRootPath); + Assert.NotEqual(0, exitCode); + Assert.True(File.Exists(_databasePath + ".before-commit.started")); + Assert.Equal(6L, await ExecuteScalarInt64Async(_databasePath, + "SELECT version FROM cfsharp_schema WHERE singleton = 1;")); + await using ICloudStateStore recovered = await CreateFactory().OpenAsync(CreateContext()); + await using ICloudStateTransaction verify = await recovered.BeginTransactionAsync(); + Assert.Equal(new byte[] { 12 }, (await verify.Checkpoints.GetAsync("retained/before-crash"))!.Value.ToArray()); + Assert.Null(await verify.Items.GetByRelativePathAsync("crash-recovery.txt")); + } + [Fact] public async Task NewerSchemaVersionIsRejected() { @@ -473,7 +558,7 @@ CREATE TABLE cfsharp_schema ( singleton INTEGER NOT NULL PRIMARY KEY CHECK (singleton = 1), version INTEGER NOT NULL ); - INSERT INTO cfsharp_schema(singleton, version) VALUES(1, 6); + INSERT INTO cfsharp_schema(singleton, version) VALUES(1, 7); """); SqliteCloudStateStoreException exception = await Assert.ThrowsAsync< From fa600f70e82913ec3737ec4d7249541634aeeb98 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:47:25 +0800 Subject: [PATCH 08/40] chore: clarify MirrorPulse Team code ownership --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 42d2526..c80733b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ -# Default owner for repository changes. Replace with an organization team when -# a dedicated MirrorPulse maintainers team is created. +# MirrorPulse Team maintains the project. Keep the individual GitHub owner until +# a visible organization team is available; team entries require @org/team-slug. * @Quaternion8192 From f050cfab9f10a26f3e8c15ded286ba5cacebe218 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:59:33 +0800 Subject: [PATCH 09/40] feat(items): prepare durable directory move proofs --- docs/placeholders.md | 14 ++ eng/api-baseline.json | 12 ++ ...loudFileSystem.DirectoryMovePreparation.cs | 73 +++++++ .../Items/CloudDirectory.MovePreparation.cs | 28 +++ src/CfSharp/Items/CloudDirectoryMoveProof.cs | 200 ++++++++++++++++++ .../Persistence/CloudDirectoryMoveEvidence.cs | 125 +++++++++++ .../CloudDirectoryMoveTests.cs | 181 ++++++++++++++++ .../CloudDirectoryMoveProofTests.cs | 80 +++++++ 8 files changed, 713 insertions(+) create mode 100644 src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs create mode 100644 src/CfSharp/Items/CloudDirectory.MovePreparation.cs create mode 100644 src/CfSharp/Items/CloudDirectoryMoveProof.cs create mode 100644 src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs create mode 100644 tests/CfSharp.Tests/Persistence/CloudDirectoryMoveProofTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index 4f71793..d8b94cc 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -1,5 +1,19 @@ # Placeholders and hydration +## Prepare directory move evidence + +Before authorizing an external rename of a managed directory, call +`await directory.PrepareMoveAsync(destination, name)` and persist the returned proof's `Encode()` +bytes. Preparation performs no native move and returns only after immutable evidence and the +known durable subtree membership commit. `CloudDirectoryMoveProof.Decode(bytes)` restores copied +metadata; it does not make caller-constructed data authoritative. Recovery must still validate the +library's original preparation in the same store and the actual complete native binding and identity. + +Each preparation has its own ID and survives restart independently. Do not substitute a binding +observed at a renamed target for missing historical evidence. Ordinary directories may use normal +`MoveToAsync`, but cannot prepare external recovery without a managed placeholder identity and +complete native IDs. Metadata capture reads no content and holds no continuing lock after returning. + Placeholder operations use immutable, kind-specific specifications rather than exposing native unions and flag combinations directly to application code. diff --git a/eng/api-baseline.json b/eng/api-baseline.json index f633a22..4623fe6 100644 --- a/eng/api-baseline.json +++ b/eng/api-baseline.json @@ -727,6 +727,7 @@ "method CfSharp.CloudFile GetFile(System.String relativePath)", "method CfSharp.CloudItem Resolve(System.String relativePath)", "method System.Collections.Generic.IAsyncEnumerable\u003CCfSharp.CloudItem\u003E EnumerateLocalChildrenAsync(CfSharp.CloudDirectoryEnumerationOptions options,System.Threading.CancellationToken cancellationToken)", + "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudDirectoryMoveProof\u003E PrepareMoveAsync(CfSharp.CloudDirectory destination,System.String name,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudPlaceholderBatchEntryResult\u003E CreatePlaceholderAsync(CfSharp.CloudPlaceholderSpec placeholder,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudPlaceholderBatchResult\u003E CreatePlaceholdersAsync(System.Collections.Generic.IEnumerable\u003CCfSharp.CloudPlaceholderSpec\u003E placeholders,CfSharp.CloudPlaceholderBatchOptions options,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudPlaceholderMutationResult\u003E SetPopulationStateAsync(CfSharp.CloudDirectoryPopulationState state,System.Threading.CancellationToken cancellationToken)", @@ -758,6 +759,17 @@ "field CfSharp.CloudDirectoryEnumerationOrder NameAscending", "field CfSharp.CloudDirectoryEnumerationOrder NameDescending", "field System.Int32 value__", + "type CfSharp.CloudDirectoryMoveProof", + "method CfSharp.CloudDirectoryMoveProof Decode(System.ReadOnlySpan\u003CSystem.Byte\u003E encoded)", + "method System.Byte[] Encode()", + "property System.String DestinationRelativePath()", + "property CfSharp.CloudLocalFileBinding ExpectedBinding()", + "property System.ReadOnlyMemory\u003CSystem.Byte\u003E ExpectedPlaceholderIdentity()", + "property System.Guid ProofId()", + "property System.Guid RootItemId()", + "property System.String SourceRelativePath()", + "property System.Guid StoreScope()", + "property System.Int32 Version()", "type CfSharp.CloudDirectoryPlaceholderSpec", "method CfSharp.CloudDirectoryPlaceholderSpec\u002BBuilder CreateBuilder(System.String name,CfSharp.CloudPlaceholderIdentity identity)", "method CfSharp.CloudDirectoryPlaceholderSpec\u002BBuilder CreateBuilder(System.String name,System.String remoteId)", diff --git a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs new file mode 100644 index 0000000..faf5f89 --- /dev/null +++ b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs @@ -0,0 +1,73 @@ +using CfSharp.Native; + +using Microsoft.Win32.SafeHandles; + +namespace CfSharp; + +public sealed partial class CloudFileSystem +{ + internal async ValueTask PrepareDirectoryMoveAsync(CloudDirectory source, + CloudDirectory destination, string name, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(destination); + string validatedName = CloudPlaceholderSpec.ValidateName(name); + if (!destination.IsOwnedBy(this)) + { + throw new ArgumentException("The destination must belong to this file system.", nameof(destination)); + } + + CloudDirectory target = destination.GetDirectory(validatedName); + if (source.RelativePath.Length == 0 || + string.Equals(source.FullPath, target.FullPath, StringComparison.Ordinal) || + IsStrictDescendant(target.FullPath, source.FullPath)) + { + throw new InvalidOperationException("The sync root, same path, or descendant cannot be selected for a move."); + } + + using CloudFileSystemOperationLease operation = await AcquireOperationAsync( + [CloudItemOperationScope.Subtree(source.FullPath), CloudItemOperationScope.Subtree(target.FullPath)], + cancellationToken).ConfigureAwait(false); + if (!CloudItemInspector.Inspect(destination.FullPath, CloudItemKind.Directory).Exists) + { + throw new DirectoryNotFoundException("The destination parent does not exist."); + } + + // A metadata-only no-delete guard stabilizes this object during preparation. It does not + // freeze descendant writes or placeholder identity changes and is never CFAPI protection. + using SafeFileHandle guard = WindowsFileMetadata.Open(source.FullPath, preventDelete: true); + DirectoryNativeObservation captured = CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), SyncRootPath); + if (!string.Equals(captured.ActualPath, source.FullPath, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("The captured directory is outside the requested namespace path."); + } + + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Decode(captured.Identity); + await using ICloudStateTransaction transaction = await operation.StateStore.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + CloudItemState? root = await transaction.Items.GetByItemIdAsync(identity.ItemId, cancellationToken).ConfigureAwait(false); + if (root is null || root.IsTombstone || root.Kind != CloudItemKind.Directory || + !string.Equals(root.RelativePath.Replace('/', '\\'), source.RelativePath.Replace('/', '\\'), StringComparison.OrdinalIgnoreCase) || + root.RemoteId != identity.RemoteId) + { + throw new InvalidOperationException("The native directory identity does not match its live durable source."); + } + + Guid scope = await CloudDirectoryMoveEvidence.GetScopeAsync(transaction, create: true, cancellationToken).ConfigureAwait(false); + string actualSourceRelativePath = Path.GetRelativePath(SyncRootPath, captured.ActualPath); + CloudDirectoryMoveProof proof = new(Guid.NewGuid(), scope, actualSourceRelativePath, target.RelativePath, + root.ItemId, captured.Binding, captured.Identity); + IReadOnlyList members = await transaction.Items.ListSubtreeAsync(source.RelativePath, cancellationToken).ConfigureAwait(false); + await CloudDirectoryMoveEvidence.PrepareAsync(transaction, proof, members, cancellationToken).ConfigureAwait(false); + if (!captured.Matches(CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), SyncRootPath))) + { + throw new InvalidOperationException("The native directory changed during preparation."); + } + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + if (!captured.Matches(CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), SyncRootPath))) + { + throw new InvalidOperationException("The native directory changed after preparation committed; capture a new proof."); + } + + return proof; + } +} diff --git a/src/CfSharp/Items/CloudDirectory.MovePreparation.cs b/src/CfSharp/Items/CloudDirectory.MovePreparation.cs new file mode 100644 index 0000000..f018b7c --- /dev/null +++ b/src/CfSharp/Items/CloudDirectory.MovePreparation.cs @@ -0,0 +1,28 @@ +namespace CfSharp; + +public sealed partial class CloudDirectory +{ + /// Captures and durably prepares evidence before this placeholder directory is moved. + /// An existing directory in this file system. + /// The intended destination child name. + /// Cancels lease acquisition, state access, or preparation before commit. + /// + /// An owned serializable proof after the library's immutable preparation and known subtree + /// membership commit. This method performs no rename. Retain the encoded proof across restart. + /// + /// + /// Requires Windows 10 version 1709, a managed live directory placeholder, complete native IDs, + /// and the owning durable store. Metadata handles guard capture but are closed before returning; + /// the returned value confers no continuing protection. Concurrent operations are coordinated + /// within this file system. Cross-process identity and descendant changes still require recovery + /// validation. Ordinary directories retain their existing normal move API. + /// + /// The destination belongs to another file system or the name is invalid. + /// The root, destination, or durable/native identity is unsuitable. + /// Storage cannot supply the complete native binding. + /// Native metadata, path protection, or durable preparation failed. + /// Preparation was canceled; no proof is returned. + public ValueTask PrepareMoveAsync(CloudDirectory destination, string name, + CancellationToken cancellationToken = default) => + Owner.PrepareDirectoryMoveAsync(this, destination, name, cancellationToken); +} diff --git a/src/CfSharp/Items/CloudDirectoryMoveProof.cs b/src/CfSharp/Items/CloudDirectoryMoveProof.cs new file mode 100644 index 0000000..22c3cde --- /dev/null +++ b/src/CfSharp/Items/CloudDirectoryMoveProof.cs @@ -0,0 +1,200 @@ +using System.Text; + +namespace CfSharp; + +/// Contains durable, pre-move evidence for one managed placeholder directory. +/// +/// This immutable value owns copied metadata only; it holds no handle, transaction, or lock. +/// Persist before authorizing an external rename. Decoding does not establish +/// authority: recovery also requires the original preparation in the same state store, the actual +/// sync-root binding, and the complete current placeholder identity. Native IDs may be reused after +/// deletion. The proof does not establish content equality or remote acceptance. Concurrent reads +/// are safe. Preparation requires Windows 10 version 1709 and file-ID-capable storage. +/// +public sealed class CloudDirectoryMoveProof +{ + private readonly byte[] _identity; + + internal CloudDirectoryMoveProof(Guid proofId, Guid storeScope, string sourceRelativePath, + string destinationRelativePath, Guid rootItemId, CloudLocalFileBinding expectedBinding, + ReadOnlySpan expectedPlaceholderIdentity) + { + if (proofId == Guid.Empty || storeScope == Guid.Empty || rootItemId == Guid.Empty) + { + throw new ArgumentException("Proof, store, and item identifiers must be nonempty."); + } + + ArgumentNullException.ThrowIfNull(expectedBinding); + SourceRelativePath = CloudDirectoryEvidenceCodec.CanonicalPath(sourceRelativePath); + DestinationRelativePath = CloudDirectoryEvidenceCodec.CanonicalPath(destinationRelativePath); + if (SourceRelativePath == DestinationRelativePath || + DestinationRelativePath.StartsWith(SourceRelativePath + "\\", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException("A move requires a different destination outside the source subtree."); + } + + if (expectedPlaceholderIdentity.Length > SyncRootRegistrationOptions.MaxFileIdentityLength || + CloudPlaceholderIdentity.Decode(expectedPlaceholderIdentity).ItemId != rootItemId) + { + throw new ArgumentException("The complete placeholder identity must identify the prepared root."); + } + + ProofId = proofId; + StoreScope = storeScope; + RootItemId = rootItemId; + ExpectedBinding = expectedBinding; + _identity = expectedPlaceholderIdentity.ToArray(); + } + + /// Gets the supported serialization protocol version. + public int Version { get; } = 1; + + /// Gets the unique identifier of this immutable preparation. + public Guid ProofId { get; } + + /// Gets the durable scope of the store that captured this proof. + public Guid StoreScope { get; } + + /// Gets the canonical source path relative to the owning sync root. + public string SourceRelativePath { get; } + + /// Gets the canonical intended destination path relative to that root. + public string DestinationRelativePath { get; } + + /// Gets the historical CfSharp identity of the directory root. + public Guid RootItemId { get; } + + /// Gets the complete volume, root, and directory object binding captured before movement. + public CloudLocalFileBinding ExpectedBinding { get; } + + /// Gets a copy of the complete opaque native placeholder identity captured before movement. + public ReadOnlyMemory ExpectedPlaceholderIdentity => _identity.ToArray(); + + /// Serializes this proof into an owned, bounded versioned envelope. + /// A new byte array suitable for durable caller storage; no private subtree manifest is included. + public byte[] Encode() + { + using MemoryStream stream = new(); + using BinaryWriter writer = new(stream, Encoding.UTF8, leaveOpen: true); + writer.Write(0x31504D43); // CMP1, little endian; all integer fields use BinaryWriter's fixed convention. + writer.Write(Version); + writer.Write(ProofId.ToByteArray()); + writer.Write(StoreScope.ToByteArray()); + writer.Write(RootItemId.ToByteArray()); + writer.Write(ExpectedBinding.VolumeSerialNumber); + writer.Write(ExpectedBinding.SyncRootFileId.ToByteArray()); + writer.Write(ExpectedBinding.LocalFileId.ToByteArray()); + CloudDirectoryEvidenceCodec.WriteText(writer, SourceRelativePath); + CloudDirectoryEvidenceCodec.WriteText(writer, DestinationRelativePath); + writer.Write(_identity.Length); + writer.Write(_identity); + return stream.ToArray(); + } + + /// Restores a serialized proof without granting trusted provenance. + /// The complete envelope, at most 131072 bytes. + /// An immutable owned value; recovery must verify the library's original durable record. + /// The envelope, paths, lengths, or identity are invalid. + /// The envelope has an unsupported protocol version. + public static CloudDirectoryMoveProof Decode(ReadOnlySpan encoded) + { + if (encoded.Length is < 108 or > 131072) + { + throw new InvalidDataException("The directory move proof has an invalid length."); + } + + try + { + using MemoryStream stream = new(encoded.ToArray(), writable: false); + using BinaryReader reader = new(stream, CloudDirectoryEvidenceCodec.Utf8); + if (reader.ReadInt32() != 0x31504D43) + { + throw new InvalidDataException("The value is not a directory move proof."); + } + + int version = reader.ReadInt32(); + if (version != 1) + { + throw new NotSupportedException($"Directory move proof version {version} is unsupported."); + } + + Guid proofId = CloudDirectoryEvidenceCodec.ReadGuid(reader); + Guid scope = CloudDirectoryEvidenceCodec.ReadGuid(reader); + Guid rootId = CloudDirectoryEvidenceCodec.ReadGuid(reader); + CloudLocalFileBinding binding = new(reader.ReadUInt64(), + CloudDirectoryEvidenceCodec.ReadGuid(reader), CloudDirectoryEvidenceCodec.ReadGuid(reader)); + string source = CloudDirectoryEvidenceCodec.ReadText(reader); + string destination = CloudDirectoryEvidenceCodec.ReadText(reader); + byte[] identity = CloudDirectoryEvidenceCodec.ReadBytes(reader, SyncRootRegistrationOptions.MaxFileIdentityLength); + if (stream.Position != stream.Length) + { + throw new InvalidDataException("The proof contains trailing data."); + } + + CloudDirectoryMoveProof proof = new(proofId, scope, source, destination, rootId, binding, identity); + if (!proof.Encode().AsSpan().SequenceEqual(encoded)) + { + throw new InvalidDataException("The proof does not use canonical serialization."); + } + + return proof; + } + catch (Exception exception) when (exception is EndOfStreamException or ArgumentException or DecoderFallbackException) + { + throw new InvalidDataException("The directory move proof is malformed.", exception); + } + } +} + +internal static class CloudDirectoryEvidenceCodec +{ + internal static readonly UTF8Encoding Utf8 = new(false, true); + + internal static string CanonicalPath(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + string canonical = CloudRemotePathValidation.Canonicalize(path, nameof(path)).Replace('/', '\\'); + if (Utf8.GetByteCount(canonical) > 32768) + { + throw new ArgumentException("The directory evidence path is too long.", nameof(path)); + } + + return canonical; + } + + internal static void WriteText(BinaryWriter writer, string value) + { + byte[] bytes = Utf8.GetBytes(value); + if (bytes.Length > 32768) + { + throw new InvalidDataException("The directory evidence path is too long."); + } + + writer.Write(bytes.Length); + writer.Write(bytes); + } + + internal static string ReadText(BinaryReader reader) => Utf8.GetString(ReadBytes(reader, 32768)); + + internal static byte[] ReadBytes(BinaryReader reader, int maximumLength) + { + int length = reader.ReadInt32(); + if (length < 0 || length > maximumLength || length > reader.BaseStream.Length - reader.BaseStream.Position) + { + throw new InvalidDataException("The directory evidence length is invalid."); + } + + return reader.ReadBytes(length); + } + + internal static Guid ReadGuid(BinaryReader reader) + { + byte[] bytes = reader.ReadBytes(16); + if (bytes.Length != 16) + { + throw new InvalidDataException("The directory evidence identifier is truncated."); + } + + return new Guid(bytes); + } +} diff --git a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs new file mode 100644 index 0000000..5327a87 --- /dev/null +++ b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs @@ -0,0 +1,125 @@ +using System.Runtime.Versioning; +using System.Text; +using CfSharp.Native; + +namespace CfSharp; + +// Evidence belongs to CfSharp, not to provider payloads. Keep each preparation independent: +// replacing a later binding must never rewrite an earlier proof or its historical membership. +internal static class CloudDirectoryMoveEvidence +{ + internal const string ScopeName = "cfsharp/namespace/scope"; + internal static string ProofName(Guid id) => $"cfsharp/namespace/preparations/{id:N}/proof"; + internal static string MembersName(Guid id) => $"cfsharp/namespace/preparations/{id:N}/members"; + + internal static async ValueTask GetScopeAsync(ICloudStateTransaction transaction, + bool create, CancellationToken cancellationToken) + { + CloudStateCheckpoint? checkpoint = await transaction.Checkpoints.GetAsync(ScopeName, cancellationToken).ConfigureAwait(false); + if (checkpoint is not null) + { + if (checkpoint.Value.Length != 20) + { + throw new InvalidDataException("The durable directory scope is malformed."); + } + + using BinaryReader reader = new(new MemoryStream(checkpoint.Value.ToArray(), writable: false)); + if (reader.ReadInt32() != 1) + { + throw new NotSupportedException("The durable directory scope version is unsupported."); + } + + Guid scope = CloudDirectoryEvidenceCodec.ReadGuid(reader); + return scope != Guid.Empty ? scope : throw new InvalidDataException("The durable directory scope is empty."); + } + + if (!create) + { + return Guid.Empty; + } + + Guid created = Guid.NewGuid(); + using MemoryStream stream = new(); + using BinaryWriter writer = new(stream); + writer.Write(1); + writer.Write(created.ToByteArray()); + await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(ScopeName, + stream.ToArray(), DateTimeOffset.UtcNow), cancellationToken).ConfigureAwait(false); + return created; + } + + [SupportedOSPlatform("windows10.0.16299")] + internal static DirectoryNativeObservation Capture(nint handle, string syncRootPath) + { + NativeFileMetadata metadata = WindowsFileMetadata.Read(handle); + if (!metadata.Directory || metadata.DeletePending || metadata.PlaceholderState == CfPlaceholderState.Invalid || + !metadata.PlaceholderState.HasFlag(CfPlaceholderState.Placeholder)) + { + throw new InvalidOperationException($"Directory recovery requires a live managed placeholder directory (directory={metadata.Directory}, deletePending={metadata.DeletePending}, state={metadata.PlaceholderState})."); + } + + _ = CloudPlaceholderIdentity.Decode(metadata.PlaceholderIdentity); + return new DirectoryNativeObservation(CloudLocalFileBindingPlatform.Read(handle, syncRootPath), + metadata.PlaceholderIdentity, WindowsFileMetadata.ReadFinalPath(handle)); + } + + internal static async ValueTask PrepareAsync(ICloudStateTransaction transaction, CloudDirectoryMoveProof proof, + IReadOnlyList members, CancellationToken cancellationToken) + { + if (await transaction.Checkpoints.GetAsync(ProofName(proof.ProofId), cancellationToken).ConfigureAwait(false) is not null || + await transaction.Checkpoints.GetAsync(MembersName(proof.ProofId), cancellationToken).ConfigureAwait(false) is not null) + { + throw new InvalidOperationException("A directory preparation is immutable."); + } + + byte[] manifest = EncodeMembers(proof, members); + DateTimeOffset now = DateTimeOffset.UtcNow; + await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(ProofName(proof.ProofId), proof.Encode(), now), + cancellationToken).ConfigureAwait(false); + await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(MembersName(proof.ProofId), manifest, now), + cancellationToken).ConfigureAwait(false); + } + + internal static byte[] EncodeMembers(CloudDirectoryMoveProof proof, IReadOnlyList members) + { + using MemoryStream stream = new(); + using BinaryWriter writer = new(stream, Encoding.UTF8, leaveOpen: true); + writer.Write(1); + writer.Write(proof.ProofId.ToByteArray()); + writer.Write(members.Count); + HashSet ids = []; + HashSet suffixes = new(StringComparer.OrdinalIgnoreCase); + bool rootPresent = false; + foreach (CloudItemState item in members) + { + string suffix = CloudDirectoryStateProjection.MapPath(item.RelativePath, proof.SourceRelativePath, string.Empty); + if (!ids.Add(item.ItemId) || !suffixes.Add(suffix) || + (suffix.Length == 0 && (item.ItemId != proof.RootItemId || item.Kind != CloudItemKind.Directory || item.IsTombstone))) + { + throw new InvalidDataException("The prepared subtree has ambiguous membership."); + } + + rootPresent |= suffix.Length == 0; + writer.Write(item.ItemId.ToByteArray()); + writer.Write((int)item.Kind); + CloudDirectoryEvidenceCodec.WriteText(writer, suffix.Replace('/', '\\')); + if (stream.Length > 64 * 1024 * 1024) + { + throw new NotSupportedException("The prepared subtree exceeds the recovery metadata limit."); + } + } + + if (!rootPresent) + { + throw new InvalidOperationException("The prepared subtree must contain its live managed directory root."); + } + + return stream.ToArray(); + } +} + +internal sealed record DirectoryNativeObservation(CloudLocalFileBinding Binding, byte[] Identity, string ActualPath) +{ + internal bool Matches(DirectoryNativeObservation other) => Binding == other.Binding && + Identity.AsSpan().SequenceEqual(other.Identity) && string.Equals(ActualPath, other.ActualPath, StringComparison.Ordinal); +} diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs new file mode 100644 index 0000000..b9cbf46 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs @@ -0,0 +1,181 @@ +using System.Runtime.Versioning; + +using CfSharp.Storage.Sqlite; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Fact] + public async Task PrepareCapturesLiveObjectAndIndependentDurableProofsWithoutMoving() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudItemSnapshot before = await source.InspectAsync(); + Assert.True(before.PlaceholderState.HasFlag(CloudPlaceholderState.Placeholder), $"Source state: {before.PlaceholderState}"); + CloudDirectoryMoveProof first = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + CloudDirectoryMoveProof second = await source.PrepareMoveAsync(root.FileSystem.Root, "Other"); + Assert.Equal(before.LocalBinding, first.ExpectedBinding); + Assert.Equal(before.ItemId, first.RootItemId); + Assert.Equal(before.PlaceholderIdentity.ToArray(), first.ExpectedPlaceholderIdentity.ToArray()); + Assert.NotEqual(first.ProofId, second.ProofId); + Assert.Equal(first.StoreScope, second.StoreScope); + Assert.True(Directory.Exists(source.FullPath)); + Assert.False(Directory.Exists(Path.Combine(root.RootPath, "Moved"))); + await using ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync(); + Assert.Equal(first.Encode(), (await transaction.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ProofName(first.ProofId)))!.Value.ToArray()); + Assert.Equal(second.Encode(), (await transaction.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ProofName(second.ProofId)))!.Value.ToArray()); + Assert.Equal(2, (await transaction.Items.ListSubtreeAsync("Docs")).Count); + } + + [Fact] + public async Task PrepareRejectsOrdinaryRootForeignDestinationCancellationAndMismatchedState() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + await using DirectoryMoveTestRoot foreign = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + await Assert.ThrowsAsync(() => source.PrepareMoveAsync(foreign.FileSystem.Root, "Moved").AsTask()); + await Assert.ThrowsAsync(() => root.FileSystem.Root.PrepareMoveAsync(root.FileSystem.Root, "Moved").AsTask()); + await Assert.ThrowsAsync(() => root.FileSystem.GetDirectory("Ordinary").PrepareMoveAsync(root.FileSystem.Root, "Moved").AsTask()); + using CancellationTokenSource cancellation = new(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => source.PrepareMoveAsync(root.FileSystem.Root, "Moved", cancellation.Token).AsTask()); + await using (ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync()) + { + await transaction.Items.RemoveAsync(root.DirectoryIdentity.ItemId); + await transaction.CommitAsync(); + } + + await Assert.ThrowsAsync(() => source.PrepareMoveAsync(root.FileSystem.Root, "Moved").AsTask()); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Empty(await verify.Checkpoints.ListAsync("cfsharp/namespace/preparations")); + Assert.True(Directory.Exists(source.FullPath)); + } + + [Fact] + public async Task PrepareCommitFailureReturnsNoProofAndLeavesSourceAndPreparationsUnchanged() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + root.Faults.BeforeCommit = () => throw new IOException("Preparation commit fault."); + await Assert.ThrowsAsync(() => root.FileSystem.GetDirectory("Docs") + .PrepareMoveAsync(root.FileSystem.Root, "Moved").AsTask()); + root.Faults.BeforeCommit = null; + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Empty(await verify.Checkpoints.ListAsync("cfsharp/namespace/preparations")); + Assert.True(Directory.Exists(Path.Combine(root.RootPath, "Docs"))); + Assert.False(Directory.Exists(Path.Combine(root.RootPath, "Moved"))); + } +} + +[SupportedOSPlatform("windows10.0.16299")] +internal sealed class DirectoryMoveTestRoot(CloudFileSystem fileSystem, DirectoryMoveTestRoot.CapturingFactory factory, + string area, CloudPlaceholderIdentity directoryIdentity) : IAsyncDisposable +{ + internal CloudFileSystem FileSystem => fileSystem; + internal string RootPath => fileSystem.SyncRootPath; + internal string Area => area; + internal string DatabasePath => Path.Combine(area, "state.db"); + internal ICloudStateStore Store => factory.Store!; + internal DirectoryMoveFaultStore Faults => factory.Store!; + internal CloudPlaceholderIdentity DirectoryIdentity => directoryIdentity; + + internal static async Task OpenAsync() + { + string area = Path.Combine(Path.GetTempPath(), "CfSharp-directory-move-tests", Guid.NewGuid().ToString("N")); + string rootPath = Path.Combine(area, "root"); + Directory.CreateDirectory(Path.Combine(rootPath, "Ordinary")); + Guid providerId = Guid.NewGuid(); + CapturingFactory factory = new(new SqliteCloudStateStoreFactory(Path.Combine(area, "state.db"))); + CloudFileSystem fileSystem = CloudFileSystem.CreateBuilder(rootPath).WithStateStore(factory) + .WithRegistration(SyncRootRegistrationOptions.CreateBuilder("CfSharp Directory Move", "1.0-test") + .WithProviderId(providerId).WithSyncRootIdentity(providerId.ToByteArray()) + .WithHydrationPolicy(CloudHydrationPolicy.Progressive).WithPopulationPolicy(CloudPopulationPolicy.Partial) + .WithRootMarkedInSync().Build()).WithContentProvider(new EmptyContentProvider()).Build(); + bool registered = false; + try + { + await fileSystem.StartAsync(); + registered = true; + CloudDirectoryPlaceholderSpec directory = CloudDirectoryPlaceholderSpec.CreateBuilder("Docs", "directory") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build(); + await fileSystem.Root.CreatePlaceholderAsync(directory); + await fileSystem.GetDirectory("Docs").CreatePlaceholderAsync(CloudFilePlaceholderSpec.CreateBuilder("child.txt", "child", 0).Build()); + return new DirectoryMoveTestRoot(fileSystem, factory, area, directory.Identity); + } + catch + { + await fileSystem.DisposeAsync(); + if (registered) + { + CloudSyncRoot.Open(rootPath).Unregister(); + } + + Directory.Delete(area, recursive: true); + throw; + } + } + + public async ValueTask DisposeAsync() + { + await fileSystem.DisposeAsync(); + CloudSyncRoot.Open(RootPath).Unregister(); + Directory.Delete(area, recursive: true); + } + + internal sealed class CapturingFactory(ICloudStateStoreFactory inner) : ICloudStateStoreFactory + { + internal DirectoryMoveFaultStore? Store { get; private set; } + public async ValueTask OpenAsync(CloudStateStoreContext context, CancellationToken cancellationToken = default) => + Store = new DirectoryMoveFaultStore(await inner.OpenAsync(context, cancellationToken)); + } + + private sealed class EmptyContentProvider : ICloudFileContentProvider + { + public ValueTask OpenReadAsync(CloudFileFetchRequest request, CancellationToken cancellationToken) => + ValueTask.FromResult(new MemoryStream()); + } +} + +internal sealed class DirectoryMoveFaultStore(ICloudStateStore inner) : ICloudStateStore +{ + internal Action? BeforeCommit { get; set; } + internal Action? AfterCommit { get; set; } + public async ValueTask BeginTransactionAsync(CancellationToken cancellationToken = default) => + new Transaction(await inner.BeginTransactionAsync(cancellationToken), this); + public ValueTask DisposeAsync() => inner.DisposeAsync(); + + private sealed class Transaction(ICloudStateTransaction transaction, DirectoryMoveFaultStore owner) : ICloudStateTransaction + { + public ICloudItemStateRepository Items => transaction.Items; + public ICloudCheckpointRepository Checkpoints => transaction.Checkpoints; + public ICloudOperationJournal Operations => transaction.Operations; + public ICloudConflictRepository Conflicts => transaction.Conflicts; + public ICloudRemoteBatchRepository RemoteBatches => transaction.RemoteBatches; + public ICloudEchoSuppressionRepository EchoSuppressions => transaction.EchoSuppressions; + public async ValueTask CommitAsync(CancellationToken cancellationToken = default) + { + owner.BeforeCommit?.Invoke(); + await transaction.CommitAsync(cancellationToken); + owner.AfterCommit?.Invoke(); + } + + public ValueTask RollbackAsync(CancellationToken cancellationToken = default) => transaction.RollbackAsync(cancellationToken); + public ValueTask DisposeAsync() => transaction.DisposeAsync(); + } +} diff --git a/tests/CfSharp.Tests/Persistence/CloudDirectoryMoveProofTests.cs b/tests/CfSharp.Tests/Persistence/CloudDirectoryMoveProofTests.cs new file mode 100644 index 0000000..c657035 --- /dev/null +++ b/tests/CfSharp.Tests/Persistence/CloudDirectoryMoveProofTests.cs @@ -0,0 +1,80 @@ +using System.Buffers.Binary; + +namespace CfSharp.Tests.Persistence; + +public sealed class CloudDirectoryMoveProofTests +{ + [Fact] + public void SerializationOwnsCompleteBindingAndIdentityAndRejectsMalformedLengths() + { + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Create("remote", "revision"); + byte[] bytes = identity.Encode(); + CloudLocalFileBinding binding = new(0x123456789abcdef0, Guid.NewGuid(), Guid.NewGuid()); + CloudDirectoryMoveProof proof = new(Guid.NewGuid(), Guid.NewGuid(), "Docs/Deep", "Target/Deep", identity.ItemId, binding, bytes); + bytes[0] = 0; + byte[] encoded = proof.Encode(); + CloudDirectoryMoveProof decoded = CloudDirectoryMoveProof.Decode(encoded); + Assert.Equal(encoded, decoded.Encode()); + Assert.Equal(binding, decoded.ExpectedBinding); + Assert.Equal(identity.Encode(), decoded.ExpectedPlaceholderIdentity.ToArray()); + Assert.Equal("Docs\\Deep", decoded.SourceRelativePath); + Assert.Equal(proof.ProofId, decoded.ProofId); + Assert.Equal(proof.StoreScope, decoded.StoreScope); + Assert.Equal(proof.RootItemId, decoded.RootItemId); + encoded[0] = 0; + Assert.Equal(proof.Encode(), decoded.Encode()); + Assert.Throws(() => CloudDirectoryMoveProof.Decode(encoded)); + Assert.Throws(() => CloudDirectoryMoveProof.Decode(new byte[131073])); + Assert.Throws(() => CloudDirectoryMoveProof.Decode(proof.Encode().AsSpan()[..^1])); + Assert.Throws(() => CloudDirectoryMoveProof.Decode([.. proof.Encode(), 0])); + byte[] corrupt = proof.Encode(); + BinaryPrimitives.WriteInt32LittleEndian(corrupt.AsSpan(96, 4), int.MaxValue); + Assert.Throws(() => CloudDirectoryMoveProof.Decode(corrupt)); + byte[] future = proof.Encode(); + BinaryPrimitives.WriteInt32LittleEndian(future.AsSpan(4, 4), 2); + Assert.Throws(() => CloudDirectoryMoveProof.Decode(future)); + } + + [Theory] + [InlineData("Docs", "Docs")] + [InlineData("Docs", "Docs\\Deep")] + [InlineData("..\\Docs", "Target")] + [InlineData("Docs", "C:\\Target")] + public void ConstructionRejectsUnsafePathShapes(string source, string destination) + { + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Create("remote"); + Assert.Throws(() => new CloudDirectoryMoveProof(Guid.NewGuid(), Guid.NewGuid(), + source, destination, identity.ItemId, new CloudLocalFileBinding(1, Guid.NewGuid(), Guid.NewGuid()), identity.Encode())); + } + + [Fact] + public async Task PreparationIsImmutableIndependentAndRollsBackWithCallerTransaction() + { + await using ICloudStateStore store = await InMemoryCloudStateStoreContractTests.CreateFactoryForTesting() + .OpenAsync(new CloudStateStoreContext(Path.GetTempPath())); + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Create("remote"); + CloudItemState root = new(identity.ItemId, "remote", "Docs", CloudItemKind.Directory, "v1", 7, false, DateTimeOffset.UtcNow); + CloudItemState tombstone = new(Guid.NewGuid(), "gone", "Docs\\gone", CloudItemKind.File, "v2", 8, true, DateTimeOffset.UtcNow); + CloudDirectoryMoveProof first; + await using (ICloudStateTransaction transaction = await store.BeginTransactionAsync()) + { + Guid scope = await CloudDirectoryMoveEvidence.GetScopeAsync(transaction, true, default); + first = new(Guid.NewGuid(), scope, "Docs", "Moved", root.ItemId, + new CloudLocalFileBinding(1, Guid.NewGuid(), Guid.NewGuid()), identity.Encode()); + await CloudDirectoryMoveEvidence.PrepareAsync(transaction, first, [root, tombstone], default); + await transaction.CommitAsync(); + } + + CloudDirectoryMoveProof second = new(Guid.NewGuid(), first.StoreScope, "Docs", "Other", root.ItemId, first.ExpectedBinding, identity.Encode()); + await using (ICloudStateTransaction rollback = await store.BeginTransactionAsync()) + { + await CloudDirectoryMoveEvidence.PrepareAsync(rollback, second, [root, tombstone], default); + } + + await using ICloudStateTransaction verify = await store.BeginTransactionAsync(); + Assert.Equal(first.StoreScope, await CloudDirectoryMoveEvidence.GetScopeAsync(verify, false, default)); + Assert.Equal(first.Encode(), (await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ProofName(first.ProofId)))!.Value.ToArray()); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ProofName(second.ProofId))); + await Assert.ThrowsAsync(() => CloudDirectoryMoveEvidence.PrepareAsync(verify, first, [root, tombstone], default).AsTask()); + } +} From 0bc8edfead254de447c1be8cf288b46eac6bd4c8 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:05:42 +0800 Subject: [PATCH 10/40] fix(items): retain directory bindings during identity projection --- docs/placeholders.md | 7 + .../CloudFileSystem.ItemMutations.cs | 8 + .../CloudFileSystem.PlaceholderOperations.cs | 6 + .../CloudFileSystem.RemoteCreation.cs | 2 + .../Persistence/CloudDirectoryMoveEvidence.cs | 9 +- .../Persistence/CloudDirectoryProvenance.cs | 142 ++++++++++++++++++ src/CfSharp/Providers/CloudProviderSession.cs | 7 + .../CloudDirectoryProvenanceTests.cs | 78 ++++++++++ .../ProviderDirectoryPopulationTests.cs | 15 +- .../RemoteCreationRecoveryTests.cs | 8 + 10 files changed, 277 insertions(+), 5 deletions(-) create mode 100644 src/CfSharp/Persistence/CloudDirectoryProvenance.cs create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryProvenanceTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index d8b94cc..141d24a 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -14,6 +14,13 @@ observed at a renamed target for missing historical evidence. Ordinary directori `MoveToAsync`, but cannot prepare external recovery without a managed placeholder identity and complete native IDs. Metadata capture reads no content and holds no continuing lock after returning. +Directory creation, conversion, identity replacement, remote creation, and provider population +retain native provenance in the same transaction as their official item rows. Adding children +refreshes known membership of captured ancestors without enumerating native subtrees. Immutable +prepared proofs remain unchanged. Removing an identity or reverting a directory removes its live +provenance. Storage without complete IDs retains existing creation behavior but cannot establish +this recovery evidence; legacy directories can explicitly prepare while still at their source. + Placeholder operations use immutable, kind-specific specifications rather than exposing native unions and flag combinations directly to application code. diff --git a/src/CfSharp/FileSystem/CloudFileSystem.ItemMutations.cs b/src/CfSharp/FileSystem/CloudFileSystem.ItemMutations.cs index 7bb8b57..aaab72a 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.ItemMutations.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.ItemMutations.cs @@ -248,6 +248,8 @@ await transaction.Items.UpsertAsync( isTombstone: false, DateTimeOffset.UtcNow), cancellationToken).ConfigureAwait(false); + await CloudDirectoryProvenance.RetainProjectionAsync(transaction, item.SyncRootPath, + item.RelativePath, item.Kind, cancellationToken).ConfigureAwait(false); validateBeforeCommit?.Invoke(); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); // Record success before transaction disposal, which can fail after a real commit. @@ -294,6 +296,12 @@ private static async ValueTask RemoveIdentityAsync( await transaction.Items.RemoveAsync(existing.ItemId, cancellationToken) .ConfigureAwait(false); + if (existing.Kind == CloudItemKind.Directory) + { + await transaction.Checkpoints.RemoveAsync(CloudDirectoryProvenance.BindingName(existing.ItemId), cancellationToken).ConfigureAwait(false); + await transaction.Checkpoints.RemoveAsync(CloudDirectoryProvenance.MembersName(existing.ItemId), cancellationToken).ConfigureAwait(false); + } + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); return true; } diff --git a/src/CfSharp/FileSystem/CloudFileSystem.PlaceholderOperations.cs b/src/CfSharp/FileSystem/CloudFileSystem.PlaceholderOperations.cs index f5d7f48..dcc8426 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.PlaceholderOperations.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.PlaceholderOperations.cs @@ -308,6 +308,12 @@ await transaction.Items.UpsertAsync( CancellationToken.None).ConfigureAwait(false); } + foreach (PlaceholderCreationWorkEntry entry in entries) + { + await CloudDirectoryProvenance.RetainProjectionAsync(transaction, entry.Item.SyncRootPath, + entry.Item.RelativePath, entry.Specification.Kind, CancellationToken.None).ConfigureAwait(false); + } + await transaction.CommitAsync(CancellationToken.None).ConfigureAwait(false); } diff --git a/src/CfSharp/FileSystem/CloudFileSystem.RemoteCreation.cs b/src/CfSharp/FileSystem/CloudFileSystem.RemoteCreation.cs index a598277..d218c40 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.RemoteCreation.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.RemoteCreation.cs @@ -93,6 +93,8 @@ await preparation.Checkpoints.UpsertAsync(new CloudStateCheckpoint(name, intent. await transaction.Items.UpsertAsync(new CloudItemState(intent.Identity.ItemId, change.RemoteId, change.RelativePath, change.ItemKind, change.RemoteRevision, localFileId: null, isTombstone: false, now), CancellationToken.None).ConfigureAwait(false); + await CloudDirectoryProvenance.RetainProjectionAsync(transaction, SyncRootPath, + change.RelativePath, change.ItemKind, CancellationToken.None).ConfigureAwait(false); if (options.SuppressLocalEcho && !intent.Committed) { await transaction.EchoSuppressions.UpsertAsync(new CloudEchoSuppressionState( diff --git a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs index 5327a87..69602ea 100644 --- a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs +++ b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs @@ -81,20 +81,23 @@ await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(MembersName(p } internal static byte[] EncodeMembers(CloudDirectoryMoveProof proof, IReadOnlyList members) + => EncodeMembers(proof.ProofId, proof.RootItemId, proof.SourceRelativePath, members); + + internal static byte[] EncodeMembers(Guid evidenceId, Guid rootItemId, string sourcePath, IReadOnlyList members) { using MemoryStream stream = new(); using BinaryWriter writer = new(stream, Encoding.UTF8, leaveOpen: true); writer.Write(1); - writer.Write(proof.ProofId.ToByteArray()); + writer.Write(evidenceId.ToByteArray()); writer.Write(members.Count); HashSet ids = []; HashSet suffixes = new(StringComparer.OrdinalIgnoreCase); bool rootPresent = false; foreach (CloudItemState item in members) { - string suffix = CloudDirectoryStateProjection.MapPath(item.RelativePath, proof.SourceRelativePath, string.Empty); + string suffix = CloudDirectoryStateProjection.MapPath(item.RelativePath, sourcePath, string.Empty); if (!ids.Add(item.ItemId) || !suffixes.Add(suffix) || - (suffix.Length == 0 && (item.ItemId != proof.RootItemId || item.Kind != CloudItemKind.Directory || item.IsTombstone))) + (suffix.Length == 0 && (item.ItemId != rootItemId || item.Kind != CloudItemKind.Directory || item.IsTombstone))) { throw new InvalidDataException("The prepared subtree has ambiguous membership."); } diff --git a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs new file mode 100644 index 0000000..267235e --- /dev/null +++ b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs @@ -0,0 +1,142 @@ +using System.ComponentModel; +using System.Runtime.Versioning; + +using CfSharp.Native; + +using Microsoft.Win32.SafeHandles; + +namespace CfSharp; + +internal sealed record CloudDirectoryProvenance(Guid EvidenceId, Guid StoreScope, Guid RootItemId, + string RelativePath, CloudLocalFileBinding Binding, byte[] Identity) +{ + internal static string BindingName(Guid id) => $"cfsharp/namespace/directories/{id:N}/binding"; + internal static string MembersName(Guid id) => $"cfsharp/namespace/directories/{id:N}/members"; + + internal byte[] Encode() + { + using MemoryStream stream = new(); + using BinaryWriter writer = new(stream); + writer.Write(1); + writer.Write(EvidenceId.ToByteArray()); + writer.Write(StoreScope.ToByteArray()); + writer.Write(RootItemId.ToByteArray()); + writer.Write(Binding.VolumeSerialNumber); + writer.Write(Binding.SyncRootFileId.ToByteArray()); + writer.Write(Binding.LocalFileId.ToByteArray()); + CloudDirectoryEvidenceCodec.WriteText(writer, RelativePath); + writer.Write(Identity.Length); + writer.Write(Identity); + return stream.ToArray(); + } + + internal static CloudDirectoryProvenance Decode(ReadOnlyMemory value) + { + if (value.Length is < 100 or > 65536) + { + throw new InvalidDataException("The directory provenance length is invalid."); + } + + try + { + using MemoryStream stream = new(value.ToArray(), writable: false); + using BinaryReader reader = new(stream); + if (reader.ReadInt32() != 1) + { + throw new NotSupportedException("The directory provenance version is unsupported."); + } + + Guid evidenceId = CloudDirectoryEvidenceCodec.ReadGuid(reader); + Guid scope = CloudDirectoryEvidenceCodec.ReadGuid(reader); + Guid rootId = CloudDirectoryEvidenceCodec.ReadGuid(reader); + CloudLocalFileBinding binding = new(reader.ReadUInt64(), CloudDirectoryEvidenceCodec.ReadGuid(reader), + CloudDirectoryEvidenceCodec.ReadGuid(reader)); + string path = CloudDirectoryEvidenceCodec.CanonicalPath(CloudDirectoryEvidenceCodec.ReadText(reader)); + byte[] identity = CloudDirectoryEvidenceCodec.ReadBytes(reader, SyncRootRegistrationOptions.MaxFileIdentityLength); + if (stream.Position != stream.Length || evidenceId == Guid.Empty || scope == Guid.Empty || + rootId == Guid.Empty || CloudPlaceholderIdentity.Decode(identity).ItemId != rootId) + { + throw new InvalidDataException("The directory provenance is malformed."); + } + + return new(evidenceId, scope, rootId, path, binding, identity); + } + catch (Exception exception) when (exception is ArgumentException or EndOfStreamException) + { + throw new InvalidDataException("The directory provenance is malformed.", exception); + } + } + + [SupportedOSPlatform("windows10.0.16299")] + internal static async ValueTask RetainProjectionAsync(ICloudStateTransaction transaction, string syncRootPath, + string relativePath, CloudItemKind kind, CancellationToken cancellationToken) + { + if (kind == CloudItemKind.Directory && relativePath.Length != 0) + { + using SafeFileHandle guard = WindowsFileMetadata.Open(Path.Combine(syncRootPath, relativePath), preventDelete: true); + DirectoryNativeObservation captured; + try + { + captured = CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), syncRootPath); + } + catch (Win32Exception exception) when (exception.NativeErrorCode is 1 or 50 or 87) + { + // Complete file IDs are optional for existing creation APIs. Do not fabricate + // recovery evidence on storage that cannot report them. + return; + } + catch (NotSupportedException) + { + return; + } + + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Decode(captured.Identity); + CloudItemState? item = await transaction.Items.GetByItemIdAsync(identity.ItemId, cancellationToken).ConfigureAwait(false); + if (item is null || item.Kind != CloudItemKind.Directory || item.IsTombstone || item.RemoteId != identity.RemoteId || + !string.Equals(item.RelativePath, relativePath, StringComparison.OrdinalIgnoreCase) || + !string.Equals(captured.ActualPath, Path.Combine(syncRootPath, relativePath), StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Directory provenance does not match the projected native item."); + } + + Guid scope = await CloudDirectoryMoveEvidence.GetScopeAsync(transaction, true, cancellationToken).ConfigureAwait(false); + CloudDirectoryProvenance provenance = new(Guid.NewGuid(), scope, item.ItemId, + Path.GetRelativePath(syncRootPath, captured.ActualPath), captured.Binding, captured.Identity); + await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(BindingName(item.ItemId), provenance.Encode(), DateTimeOffset.UtcNow), + cancellationToken).ConfigureAwait(false); + await RefreshMembersAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); + } + + // Refresh known membership of already captured ancestors using official transactional + // rows, without traversing native subtrees or manufacturing a new historical binding. + string? parent = Path.GetDirectoryName(relativePath); + while (!string.IsNullOrEmpty(parent)) + { + CloudItemState? directory = await transaction.Items.GetByRelativePathAsync(parent, cancellationToken).ConfigureAwait(false); + if (directory is { Kind: CloudItemKind.Directory, IsTombstone: false }) + { + CloudStateCheckpoint? checkpoint = await transaction.Checkpoints.GetAsync(BindingName(directory.ItemId), cancellationToken).ConfigureAwait(false); + if (checkpoint is not null) + { + CloudDirectoryProvenance provenance = Decode(checkpoint.Value); + if (provenance.RootItemId == directory.ItemId && + string.Equals(provenance.RelativePath, parent, StringComparison.OrdinalIgnoreCase)) + { + await RefreshMembersAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); + } + } + } + + parent = Path.GetDirectoryName(parent); + } + } + + private static async ValueTask RefreshMembersAsync(ICloudStateTransaction transaction, CloudDirectoryProvenance provenance, + CancellationToken cancellationToken) + { + IReadOnlyList members = await transaction.Items.ListSubtreeAsync(provenance.RelativePath, cancellationToken).ConfigureAwait(false); + await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(MembersName(provenance.RootItemId), + CloudDirectoryMoveEvidence.EncodeMembers(provenance.EvidenceId, provenance.RootItemId, provenance.RelativePath, members), DateTimeOffset.UtcNow), + cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/CfSharp/Providers/CloudProviderSession.cs b/src/CfSharp/Providers/CloudProviderSession.cs index 9ed12e4..deca0d5 100644 --- a/src/CfSharp/Providers/CloudProviderSession.cs +++ b/src/CfSharp/Providers/CloudProviderSession.cs @@ -923,6 +923,13 @@ await transaction.Items.UpsertAsync( _shutdown.Token).ConfigureAwait(false); } + foreach (CloudPlaceholderSpec child in page.Children) + { + string relativePath = relativeDirectory.Length == 0 ? child.Name : Path.Combine(relativeDirectory, child.Name); + await CloudDirectoryProvenance.RetainProjectionAsync(transaction, _syncRootPath, + relativePath, child.Kind, _shutdown.Token).ConfigureAwait(false); + } + string checkpointName = GetDirectoryCheckpointName(request.NormalizedPath); if (page.ContinuationToken is null) { diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryProvenanceTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryProvenanceTests.cs new file mode 100644 index 0000000..d2c0f5a --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryProvenanceTests.cs @@ -0,0 +1,78 @@ +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Fact] + public async Task CreationAndIdentityMutationRetainBindingsWithoutRewritingPreparedProofs() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory directory = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await directory.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + CloudDirectoryProvenance before; + await using (ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync()) + { + before = CloudDirectoryProvenance.Decode((await transaction.Checkpoints.GetAsync( + CloudDirectoryProvenance.BindingName(proof.RootItemId)))!.Value); + Assert.Equal(proof.ExpectedBinding, before.Binding); + Assert.Equal(proof.ExpectedPlaceholderIdentity.ToArray(), before.Identity); + Assert.Equal(proof.StoreScope, before.StoreScope); + using BinaryReader reader = new(new MemoryStream((await transaction.Checkpoints.GetAsync( + CloudDirectoryProvenance.MembersName(proof.RootItemId)))!.Value.ToArray())); + Assert.Equal(1, reader.ReadInt32()); + Assert.Equal(before.EvidenceId, CloudDirectoryEvidenceCodec.ReadGuid(reader)); + Assert.Equal(2, reader.ReadInt32()); + } + + CloudPlaceholderIdentity replacement = new(root.DirectoryIdentity.ItemId, "directory", "new-revision"); + await directory.UpdatePlaceholderAsync(CloudPlaceholderPatch.CreateBuilder().WithIdentity(replacement).Build()); + await using (ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync()) + { + CloudDirectoryProvenance after = CloudDirectoryProvenance.Decode((await transaction.Checkpoints.GetAsync( + CloudDirectoryProvenance.BindingName(proof.RootItemId)))!.Value); + Assert.Equal(before.Binding, after.Binding); + Assert.Equal(replacement.Encode(), after.Identity); + Assert.NotEqual(before.EvidenceId, after.EvidenceId); + Assert.Equal(proof.Encode(), (await transaction.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ProofName(proof.ProofId)))!.Value.ToArray()); + } + + await directory.UpdatePlaceholderAsync(CloudPlaceholderPatch.CreateBuilder().WithIdentityRemoval().Build()); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(proof.RootItemId))); + Assert.NotNull(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ProofName(proof.ProofId))); + } + + [Fact] + public async Task DirectoryConversionAndRevertRetainOnlyLiveProvenance() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory directory = root.FileSystem.GetDirectory("Ordinary"); + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Create("converted"); + await directory.ConvertToPlaceholderAsync(identity); + CloudItemSnapshot snapshot = await directory.InspectAsync(); + await using (ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync()) + { + CloudDirectoryProvenance provenance = CloudDirectoryProvenance.Decode((await transaction.Checkpoints.GetAsync( + CloudDirectoryProvenance.BindingName(identity.ItemId)))!.Value); + Assert.Equal(snapshot.LocalBinding, provenance.Binding); + Assert.Equal(identity.Encode(), provenance.Identity); + } + + await directory.RevertToRegularItemAsync(); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(identity.ItemId))); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.MembersName(identity.ItemId))); + } +} diff --git a/tests/CfSharp.IntegrationTests/ProviderDirectoryPopulationTests.cs b/tests/CfSharp.IntegrationTests/ProviderDirectoryPopulationTests.cs index 7d48090..2f6ae50 100644 --- a/tests/CfSharp.IntegrationTests/ProviderDirectoryPopulationTests.cs +++ b/tests/CfSharp.IntegrationTests/ProviderDirectoryPopulationTests.cs @@ -24,6 +24,7 @@ public async Task PartialDirectoryRequestsAnOrderedProviderPage() CloudSyncRoot? root = null; bool registered = false; DemandProvider provider = new(); + DirectoryMoveTestRoot.CapturingFactory factory = new(new SqliteCloudStateStoreFactory(databasePath)); try { @@ -37,7 +38,7 @@ public async Task PartialDirectoryRequestsAnOrderedProviderPage() .Build(); fileSystem = CloudFileSystem.CreateBuilder(rootPath) - .WithStateStore(new SqliteCloudStateStoreFactory(databasePath)) + .WithStateStore(factory) .WithRegistration(registration) .WithContentProvider(provider) .Build(); @@ -85,6 +86,14 @@ await fileSystem.GetDirectory("remote") Assert.Equal("remote-child", childSnapshot.RemoteId); Assert.NotNull(childSnapshot.DurableStateUpdatedAt); Assert.Equal(1, provider.RequestCount); + CloudItemSnapshot childDirectory = await fileSystem.GetDirectory("remote/folder").InspectAsync(); + await using (ICloudStateTransaction transaction = await factory.Store!.BeginTransactionAsync()) + { + CloudDirectoryProvenance provenance = CloudDirectoryProvenance.Decode((await transaction.Checkpoints.GetAsync( + CloudDirectoryProvenance.BindingName(childDirectory.ItemId!.Value)))!.Value); + Assert.Equal(childDirectory.LocalBinding, provenance.Binding); + Assert.Equal(childDirectory.PlaceholderIdentity.ToArray(), provenance.Identity); + } await fileSystem.DisposeAsync(); fileSystem = null; @@ -164,7 +173,9 @@ public ValueTask FetchChildrenAsync( .CreateBuilder("child.txt", "remote-child", 3) .WithInSyncState(false) .Build(); - return ValueTask.FromResult(new CloudProviderDirectoryPage([child], totalCount: 1)); + CloudDirectoryPlaceholderSpec directory = CloudDirectoryPlaceholderSpec.CreateBuilder("folder", "remote-folder") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build(); + return ValueTask.FromResult(new CloudProviderDirectoryPage([child, directory], totalCount: 2)); } } } diff --git a/tests/CfSharp.IntegrationTests/RemoteCreationRecoveryTests.cs b/tests/CfSharp.IntegrationTests/RemoteCreationRecoveryTests.cs index a28e365..819207f 100644 --- a/tests/CfSharp.IntegrationTests/RemoteCreationRecoveryTests.cs +++ b/tests/CfSharp.IntegrationTests/RemoteCreationRecoveryTests.cs @@ -128,6 +128,14 @@ public async Task NewDirectoryBatchAndFailedCreationCommitReplayWithStableIdenti Assert.Equal(2, (await complete.Items.ListSubtreeAsync(string.Empty)).Count); Assert.All(await complete.EchoSuppressions.ListActiveAsync(DateTimeOffset.UtcNow), suppression => Assert.NotNull(suppression.ItemId)); + if (!files) + { + CloudDirectoryProvenance provenance = CloudDirectoryProvenance.Decode((await complete.Checkpoints.GetAsync( + CloudDirectoryProvenance.BindingName(nativeId)))!.Value); + Assert.Equal(nativeId, provenance.RootItemId); + Assert.Equal("one", provenance.RelativePath); + Assert.Equal(CloudPlaceholderIdentity.Decode(provenance.Identity).ItemId, nativeId); + } } finally { From 444b29dea0584cd1252f536469db0dfc65178e6f Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:24:30 +0800 Subject: [PATCH 11/40] fix(items): reconcile verified external directory moves --- docs/placeholders.md | 16 ++ eng/api-baseline.json | 29 ++ ...dFileSystem.DirectoryMoveReconciliation.cs | 250 ++++++++++++++++++ .../CloudDirectory.MoveReconciliation.cs | 24 ++ .../CloudDirectoryMoveReconciliationResult.cs | 82 ++++++ .../Persistence/CloudDirectoryMoveEvidence.cs | 86 ++++++ .../Persistence/CloudDirectoryProvenance.cs | 39 +++ .../CloudDirectoryStateProjection.cs | 89 +++++++ .../CloudDirectoryMoveReconciliationTests.cs | 202 ++++++++++++++ .../VerifiedDirectoryProjectionTests.cs | 99 +++++++ 10 files changed, 916 insertions(+) create mode 100644 src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs create mode 100644 src/CfSharp/Items/CloudDirectory.MoveReconciliation.cs create mode 100644 src/CfSharp/Items/CloudDirectoryMoveReconciliationResult.cs create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveReconciliationTests.cs create mode 100644 tests/CfSharp.Tests/Persistence/VerifiedDirectoryProjectionTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index 141d24a..0d16ff9 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -21,6 +21,22 @@ prepared proofs remain unchanged. Removing an identity or reverting a directory provenance. Storage without complete IDs retains existing creation behavior but cannot establish this recovery evidence; legacy directories can explicitly prepare while still at their source. +After the external move, call `await originalDirectory.ReconcileMoveAsync(proof)` on the original +source reference. Recovery never issues another native move. It validates the original stored +preparation, complete native root/directory binding and opaque identity, actual namespace spelling, +and known current membership. Source and destination paths and an immutable completion receipt +commit together. Current revisions, local IDs, tombstones, and pending journal IDs/sequences/payloads +are retained; a root already at the target is still a proven member. Unrelated target rows remain +untouched and cause a conflict. A historical receipt cannot regress later state. + +Inspect `Outcome`, `Stage`, `NativeMoveObserved`, `DurableProjectionCommitted`, and +`RequiresFullRescan`. A store failure can leave a verified native move pending; retry its original +proof. `Error` preserves the underlying failure and `NativeHResult` is populated only for actual +native observation failures. Metadata guards stabilize the root object but do not freeze descendant +changes or identity writes. Validation around commit detects changed known descendants; local +ordinary children without native placeholder identity require a rescan before dispatch. A detected +post-commit race reports the committed fact and rescan requirement rather than claiming rollback. + Placeholder operations use immutable, kind-specific specifications rather than exposing native unions and flag combinations directly to application code. diff --git a/eng/api-baseline.json b/eng/api-baseline.json index 4623fe6..fd63126 100644 --- a/eng/api-baseline.json +++ b/eng/api-baseline.json @@ -728,6 +728,7 @@ "method CfSharp.CloudItem Resolve(System.String relativePath)", "method System.Collections.Generic.IAsyncEnumerable\u003CCfSharp.CloudItem\u003E EnumerateLocalChildrenAsync(CfSharp.CloudDirectoryEnumerationOptions options,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudDirectoryMoveProof\u003E PrepareMoveAsync(CfSharp.CloudDirectory destination,System.String name,System.Threading.CancellationToken cancellationToken)", + "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudDirectoryMoveReconciliationResult\u003E ReconcileMoveAsync(CfSharp.CloudDirectoryMoveProof proof,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudPlaceholderBatchEntryResult\u003E CreatePlaceholderAsync(CfSharp.CloudPlaceholderSpec placeholder,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudPlaceholderBatchResult\u003E CreatePlaceholdersAsync(System.Collections.Generic.IEnumerable\u003CCfSharp.CloudPlaceholderSpec\u003E placeholders,CfSharp.CloudPlaceholderBatchOptions options,System.Threading.CancellationToken cancellationToken)", "method System.Threading.Tasks.ValueTask\u003CCfSharp.CloudPlaceholderMutationResult\u003E SetPopulationStateAsync(CfSharp.CloudDirectoryPopulationState state,System.Threading.CancellationToken cancellationToken)", @@ -770,6 +771,34 @@ "property System.String SourceRelativePath()", "property System.Guid StoreScope()", "property System.Int32 Version()", + "type CfSharp.CloudDirectoryMoveReconciliationOutcome", + "field CfSharp.CloudDirectoryMoveReconciliationOutcome AlreadyProjected", + "field CfSharp.CloudDirectoryMoveReconciliationOutcome Busy", + "field CfSharp.CloudDirectoryMoveReconciliationOutcome Canceled", + "field CfSharp.CloudDirectoryMoveReconciliationOutcome Conflict", + "field CfSharp.CloudDirectoryMoveReconciliationOutcome Failed", + "field CfSharp.CloudDirectoryMoveReconciliationOutcome NativeObservedProjectionPending", + "field CfSharp.CloudDirectoryMoveReconciliationOutcome NotApplicable", + "field CfSharp.CloudDirectoryMoveReconciliationOutcome NotMoved", + "field CfSharp.CloudDirectoryMoveReconciliationOutcome Projected", + "field System.Int32 value__", + "type CfSharp.CloudDirectoryMoveReconciliationResult", + "property System.Boolean DurableProjectionCommitted()", + "property System.Int32 DurableStateEntriesUpdated()", + "property System.Exception Error()", + "property System.Nullable\u003CSystem.Int32\u003E NativeHResult()", + "property System.Boolean NativeMoveObserved()", + "property CfSharp.CloudDirectoryMoveReconciliationOutcome Outcome()", + "property System.Boolean RequiresFullRescan()", + "property CfSharp.CloudDirectoryMoveReconciliationStage Stage()", + "type CfSharp.CloudDirectoryMoveReconciliationStage", + "field CfSharp.CloudDirectoryMoveReconciliationStage Acquisition", + "field CfSharp.CloudDirectoryMoveReconciliationStage Completed", + "field CfSharp.CloudDirectoryMoveReconciliationStage CompletionValidation", + "field CfSharp.CloudDirectoryMoveReconciliationStage DurableProjection", + "field CfSharp.CloudDirectoryMoveReconciliationStage NativeObservation", + "field CfSharp.CloudDirectoryMoveReconciliationStage PreparationValidation", + "field System.Int32 value__", "type CfSharp.CloudDirectoryPlaceholderSpec", "method CfSharp.CloudDirectoryPlaceholderSpec\u002BBuilder CreateBuilder(System.String name,CfSharp.CloudPlaceholderIdentity identity)", "method CfSharp.CloudDirectoryPlaceholderSpec\u002BBuilder CreateBuilder(System.String name,System.String remoteId)", diff --git a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs new file mode 100644 index 0000000..939676d --- /dev/null +++ b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs @@ -0,0 +1,250 @@ +using System.ComponentModel; + +using CfSharp.Native; + +using Microsoft.Win32.SafeHandles; + +namespace CfSharp; + +public sealed partial class CloudFileSystem +{ + internal async ValueTask ReconcileDirectoryMoveAsync(CloudDirectory source, + CloudDirectoryMoveProof proof, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(proof); + if (!CloudDirectoryStateProjection.SamePath(source.RelativePath, proof.SourceRelativePath)) + { + throw new ArgumentException("Recovery must be called on the original source reference.", nameof(proof)); + } + + CloudDirectoryMoveReconciliationStage stage = CloudDirectoryMoveReconciliationStage.Acquisition; + bool observed = false; + bool committed = false; + int updatedCount = 0; + string? durableRootPath = null; + try + { + CloudDirectory target = GetDirectory(proof.DestinationRelativePath); + using CloudFileSystemOperationLease operation = await AcquireOperationAsync( + [CloudItemOperationScope.Subtree(source.FullPath), CloudItemOperationScope.Subtree(target.FullPath)], + cancellationToken).ConfigureAwait(false); + stage = CloudDirectoryMoveReconciliationStage.PreparationValidation; + await using (ICloudStateTransaction authentication = await operation.StateStore.BeginTransactionAsync(cancellationToken).ConfigureAwait(false)) + { + if (await CloudDirectoryMoveEvidence.AuthenticateAsync(authentication, proof, cancellationToken).ConfigureAwait(false) is null) + { + return Result(CloudDirectoryMoveReconciliationOutcome.NotApplicable); + } + + durableRootPath = (await authentication.Items.GetByItemIdAsync(proof.RootItemId, cancellationToken).ConfigureAwait(false))?.RelativePath; + await authentication.RollbackAsync(cancellationToken).ConfigureAwait(false); + } + + stage = CloudDirectoryMoveReconciliationStage.NativeObservation; + DirectoryNativeObservation? atSource = ObserveMoveNative(() => TryObserveDirectory(source.FullPath, SyncRootPath)); + if (atSource is not null && MatchesProof(atSource, proof) && + string.Equals(Path.GetRelativePath(SyncRootPath, atSource.ActualPath), proof.SourceRelativePath, StringComparison.Ordinal)) + { + stage = CloudDirectoryMoveReconciliationStage.Completed; + return Result(CloudDirectoryMoveReconciliationOutcome.NotMoved, + durableRootPath is null || !CloudDirectoryStateProjection.SamePath(durableRootPath, proof.SourceRelativePath)); + } + + using SafeFileHandle guard = ObserveMoveNative(() => WindowsFileMetadata.Open(target.FullPath, preventDelete: true)); + DirectoryNativeObservation atTarget = ObserveMoveNative(() => CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), SyncRootPath)); + if (!MatchesProof(atTarget, proof) || + !string.Equals(Path.GetRelativePath(SyncRootPath, atTarget.ActualPath), proof.DestinationRelativePath, StringComparison.Ordinal)) + { + return Result(CloudDirectoryMoveReconciliationOutcome.Conflict, requiresRescan: true); + } + + observed = true; + if (atSource is not null && !MatchesProof(atSource, proof)) + { + return Result(CloudDirectoryMoveReconciliationOutcome.Conflict, requiresRescan: true); + } + + stage = CloudDirectoryMoveReconciliationStage.DurableProjection; + VerifiedDirectoryProjection projection; + List<(string Path, NativeFileMetadata Metadata)> children; + bool unprovenLocalMembers; + await using (ICloudStateTransaction transaction = await operation.StateStore.BeginTransactionAsync(cancellationToken).ConfigureAwait(false)) + { + IReadOnlyList? members = await CloudDirectoryMoveEvidence.AuthenticateAsync(transaction, proof, cancellationToken).ConfigureAwait(false); + if (members is null) + { + return Result(CloudDirectoryMoveReconciliationOutcome.Conflict, requiresRescan: true); + } + + projection = await CloudDirectoryStateProjection.ProjectVerifiedAsync(transaction, proof, members, + DateTimeOffset.UtcNow, cancellationToken).ConfigureAwait(false); + committed = projection.AlreadyCompleted; + (children, unprovenLocalMembers) = ObserveMoveNative(() => ObserveDirectoryMembers(proof, members, projection.CurrentItems)); + if (!projection.AlreadyCompleted) + { + await CloudDirectoryProvenance.ProjectPathsAsync(transaction, proof, members, projection.CurrentItems, cancellationToken).ConfigureAwait(false); + } + + ObserveMoveNative(() => { ValidateDirectoryObservation(guard, atTarget, children); return true; }); + if (projection.AlreadyCompleted) + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + committed = true; + } + else + { + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + committed = true; + updatedCount = projection.UpdatedCount; + } + } + + stage = CloudDirectoryMoveReconciliationStage.CompletionValidation; + ObserveMoveNative(() => { ValidateDirectoryObservation(guard, atTarget, children); return true; }); + stage = CloudDirectoryMoveReconciliationStage.Completed; + return Result(projection.AlreadyCompleted ? CloudDirectoryMoveReconciliationOutcome.AlreadyProjected : + CloudDirectoryMoveReconciliationOutcome.Projected, unprovenLocalMembers); + } + catch (OperationCanceledException exception) + { + return Result(CloudDirectoryMoveReconciliationOutcome.Canceled, observed && !committed, exception); + } + catch (CloudDirectoryProjectionConflictException exception) + { + return Result(CloudDirectoryMoveReconciliationOutcome.Conflict, true, exception); + } + catch (NotSupportedException exception) when (stage is CloudDirectoryMoveReconciliationStage.PreparationValidation or CloudDirectoryMoveReconciliationStage.NativeObservation) + { + return Result(CloudDirectoryMoveReconciliationOutcome.NotApplicable, observed, exception); + } + catch (CloudDirectoryNativeObservationException exception) + { + CloudDirectoryMoveReconciliationOutcome outcome = exception.Win32ErrorCode is 32 or 33 + ? CloudDirectoryMoveReconciliationOutcome.Busy : exception.Win32ErrorCode is 2 or 3 + ? CloudDirectoryMoveReconciliationOutcome.Conflict : CloudDirectoryMoveReconciliationOutcome.Failed; + return Result(outcome, observed || outcome == CloudDirectoryMoveReconciliationOutcome.Conflict, + exception.InnerException, exception.HResult); + } + catch (InvalidOperationException exception) when (stage == CloudDirectoryMoveReconciliationStage.NativeObservation) + { + return Result(CloudDirectoryMoveReconciliationOutcome.Conflict, true, exception); + } + catch (Exception exception) + { + return Result(observed && !committed ? CloudDirectoryMoveReconciliationOutcome.NativeObservedProjectionPending : + CloudDirectoryMoveReconciliationOutcome.Failed, observed, exception); + } + + CloudDirectoryMoveReconciliationResult Result(CloudDirectoryMoveReconciliationOutcome outcome, + bool requiresRescan = false, Exception? error = null, int? nativeHResult = null) => + new(outcome, stage, observed, committed, requiresRescan, error, nativeHResult, updatedCount); + } + + private static bool MatchesProof(DirectoryNativeObservation observation, CloudDirectoryMoveProof proof) => + observation.Binding == proof.ExpectedBinding && observation.Identity.AsSpan().SequenceEqual(proof.ExpectedPlaceholderIdentity.Span); + + private static T ObserveMoveNative(Func observe) + { + try + { + return observe(); + } + catch (Win32Exception exception) + { + throw new CloudDirectoryNativeObservationException(exception, + unchecked((int)(0x80070000u | (uint)exception.NativeErrorCode)), exception.NativeErrorCode); + } + catch (NativeFileException exception) + { + throw new CloudDirectoryNativeObservationException(exception, exception.HResult, null); + } + } + + private static DirectoryNativeObservation? TryObserveDirectory(string path, string syncRootPath) + { + try + { + using SafeFileHandle handle = WindowsFileMetadata.Open(path); + return CloudDirectoryMoveEvidence.Capture(handle.DangerousGetHandle(), syncRootPath); + } + catch (Win32Exception exception) when (exception.NativeErrorCode is 2 or 3) + { + return null; + } + } + + private (List<(string Path, NativeFileMetadata Metadata)> Observations, bool RequiresRescan) ObserveDirectoryMembers( + CloudDirectoryMoveProof proof, IReadOnlyList members, IReadOnlyList current) + { + Dictionary known = members.ToDictionary(member => member.ItemId); + List<(string Path, NativeFileMetadata Metadata)> observations = []; + bool requiresRescan = false; + foreach (CloudItemState item in current.Where(item => item.ItemId != proof.RootItemId)) + { + string path = Path.Combine(SyncRootPath, CloudDirectoryStateProjection.MemberPath(proof.DestinationRelativePath, known[item.ItemId].Suffix)); + using CloudPathHandleLease parents = CloudPathHandleLease.OpenParentChains(SyncRootPath, [path]); + NativeFileMetadata metadata; + try + { + using SafeFileHandle handle = WindowsFileMetadata.Open(path); + metadata = WindowsFileMetadata.Read(handle.DangerousGetHandle()); + } + catch (Win32Exception exception) when (item.IsTombstone && exception.NativeErrorCode is 2 or 3) + { + continue; + } + + if (item.IsTombstone) + { + requiresRescan = true; + continue; + } + + if (metadata.DeletePending || metadata.Directory != (item.Kind == CloudItemKind.Directory) || + (metadata.PlaceholderIdentity.Length != 0 && CloudPlaceholderIdentity.Decode(metadata.PlaceholderIdentity).ItemId != item.ItemId) || + (metadata.PlaceholderIdentity.Length == 0 && !item.RemoteId.StartsWith("local:", StringComparison.Ordinal))) + { + throw new CloudDirectoryProjectionConflictException("A native descendant no longer matches its known durable membership."); + } + + requiresRescan |= metadata.PlaceholderIdentity.Length == 0; + observations.Add((path, metadata)); + } + + return (observations, requiresRescan); + } + + private void ValidateDirectoryObservation(SafeFileHandle guard, DirectoryNativeObservation expected, + List<(string Path, NativeFileMetadata Metadata)> children) + { + if (!expected.Matches(CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), SyncRootPath))) + { + throw new CloudDirectoryProjectionConflictException("The native directory changed during projection."); + } + + foreach ((string path, NativeFileMetadata original) in children) + { + using CloudPathHandleLease parents = CloudPathHandleLease.OpenParentChains(SyncRootPath, [path]); + using SafeFileHandle handle = WindowsFileMetadata.Open(path); + NativeFileMetadata actual = WindowsFileMetadata.Read(handle.DangerousGetHandle()); + if (original.Identity.VolumeSerialNumber != actual.Identity.VolumeSerialNumber || original.Identity.FileId != actual.Identity.FileId || + original.Directory != actual.Directory || actual.DeletePending || + !original.PlaceholderIdentity.AsSpan().SequenceEqual(actual.PlaceholderIdentity)) + { + throw new CloudDirectoryProjectionConflictException("A native descendant changed during projection; reconcile the retained journal."); + } + } + } +} + +internal sealed class CloudDirectoryNativeObservationException : IOException +{ + internal CloudDirectoryNativeObservationException(Exception inner, int hresult, int? win32ErrorCode) + : base("Native directory observation failed.", inner) + { + HResult = hresult; + Win32ErrorCode = win32ErrorCode; + } + + internal int? Win32ErrorCode { get; } +} diff --git a/src/CfSharp/Items/CloudDirectory.MoveReconciliation.cs b/src/CfSharp/Items/CloudDirectory.MoveReconciliation.cs new file mode 100644 index 0000000..1a5dd4b --- /dev/null +++ b/src/CfSharp/Items/CloudDirectory.MoveReconciliation.cs @@ -0,0 +1,24 @@ +namespace CfSharp; + +public sealed partial class CloudDirectory +{ + /// Reconciles the known durable subtree after a move using its original prepared proof. + /// The proof returned before movement, or its exact decoded serialization. + /// Cancels acquisition, validation, or storage before commit. + /// A result reporting native observation, actual durable completion, failures, and rescan requirements. + /// + /// Call on the original source reference, including after restart. Requires Windows 10 version + /// 1709 and complete native IDs. This method never moves a directory, rewrites native identity, + /// hydrates content, clears dirty state, or acknowledges journal entries. It authenticates the + /// proof in the same durable store, protects parent paths and the target with metadata guards, + /// and validates native observations around one atomic path/receipt transaction. Such guards + /// do not freeze content, identity, or descendant changes across processes. Retry a pending + /// projection with the same proof; conflicts require reconciliation, not a guessed replacement. + /// The result owns no protection lifetime. Concurrent calls are coordinated within this owner. + /// + /// The proof is null. + /// This reference is not the proof's original source path. + public ValueTask ReconcileMoveAsync(CloudDirectoryMoveProof proof, + CancellationToken cancellationToken = default) => + Owner.ReconcileDirectoryMoveAsync(this, proof, cancellationToken); +} diff --git a/src/CfSharp/Items/CloudDirectoryMoveReconciliationResult.cs b/src/CfSharp/Items/CloudDirectoryMoveReconciliationResult.cs new file mode 100644 index 0000000..90a9082 --- /dev/null +++ b/src/CfSharp/Items/CloudDirectoryMoveReconciliationResult.cs @@ -0,0 +1,82 @@ +namespace CfSharp; + +/// Describes the observed outcome of directory move recovery without issuing a native move. +public enum CloudDirectoryMoveReconciliationOutcome +{ + /// The prepared object remains at its source; nothing was projected. + NotMoved = 0, + /// The verified destination and known durable subtree were coordinated in this call. + Projected = 1, + /// The verified destination, current rows, and immutable completion receipt already agree. + AlreadyProjected = 2, + /// The native move was verified, but durable projection has not committed; retry the same proof. + NativeObservedProjectionPending = 3, + /// Native identity or durable membership conflicts with the historical evidence. + Conflict = 4, + /// The proof does not belong to this store or supported recovery scope. + NotApplicable = 5, + /// Cancellation interrupted the reported stage. + Canceled = 6, + /// A native sharing or locking conflict prevented metadata protection. + Busy = 7, + /// Another failure prevented completion at the reported stage. + Failed = 8, +} + +/// Identifies the last phase reached by directory move recovery. +public enum CloudDirectoryMoveReconciliationStage +{ + /// Acquiring the lifecycle, path scopes, and parent guards. + Acquisition = 0, + /// Authenticating the immutable preparation in its durable store. + PreparationValidation = 1, + /// Observing and guarding the actual native namespace object. + NativeObservation = 2, + /// Preflighting and committing current item paths and the receipt. + DurableProjection = 3, + /// Validating native observations after the durable commit. + CompletionValidation = 4, + /// The call completed successfully or observed that no move occurred. + Completed = 5, +} + +/// Reports directory recovery facts, including partial completion and original failures. +/// +/// Owns no handle or transaction. Immutable properties are safe for concurrent reads, although +/// the returned exception may contain mutable inherited data. Native movement and storage cannot +/// share an ACID transaction: inspect both fact flags and the rescan requirement before proceeding. +/// No outcome establishes content equality, clears dirty state, or acknowledges local operations. +/// +public sealed class CloudDirectoryMoveReconciliationResult +{ + internal CloudDirectoryMoveReconciliationResult(CloudDirectoryMoveReconciliationOutcome outcome, + CloudDirectoryMoveReconciliationStage stage, bool nativeMoveObserved, bool durableProjectionCommitted, + bool requiresFullRescan = false, Exception? error = null, int? nativeHResult = null, int updatedCount = 0) + { + Outcome = outcome; + Stage = stage; + NativeMoveObserved = nativeMoveObserved; + DurableProjectionCommitted = durableProjectionCommitted; + RequiresFullRescan = requiresFullRescan; + Error = error; + NativeHResult = nativeHResult; + DurableStateEntriesUpdated = updatedCount; + } + + /// Gets the classified recovery outcome. + public CloudDirectoryMoveReconciliationOutcome Outcome { get; } + /// Gets the last phase reached, including the phase of a failure. + public CloudDirectoryMoveReconciliationStage Stage { get; } + /// Gets whether this call verified the prepared native object at its intended destination. + public bool NativeMoveObserved { get; } + /// Gets whether current item paths and their completion receipt are durably coordinated. + public bool DurableProjectionCommitted { get; } + /// Gets whether uncertainty requires a full namespace reconciliation before dispatch. + public bool RequiresFullRescan { get; } + /// Gets the original managed or native failure, without substituting a generic storage HRESULT. + public Exception? Error { get; } + /// Gets an actual native failure HRESULT, or null for logical conflicts, cancellation, and storage failures. + public int? NativeHResult { get; } + /// Gets the number of official item paths changed in this call's committed transaction. + public int DurableStateEntriesUpdated { get; } +} diff --git a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs index 69602ea..64c2d4d 100644 --- a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs +++ b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs @@ -11,6 +11,90 @@ internal static class CloudDirectoryMoveEvidence internal const string ScopeName = "cfsharp/namespace/scope"; internal static string ProofName(Guid id) => $"cfsharp/namespace/preparations/{id:N}/proof"; internal static string MembersName(Guid id) => $"cfsharp/namespace/preparations/{id:N}/members"; + internal static string ReceiptName(Guid id) => $"cfsharp/namespace/preparations/{id:N}/receipt"; + + internal static async ValueTask?> AuthenticateAsync(ICloudStateTransaction transaction, + CloudDirectoryMoveProof proof, CancellationToken cancellationToken) + { + if (await GetScopeAsync(transaction, false, cancellationToken).ConfigureAwait(false) != proof.StoreScope) + { + return null; + } + + CloudStateCheckpoint? original = await transaction.Checkpoints.GetAsync(ProofName(proof.ProofId), cancellationToken).ConfigureAwait(false); + if (original is null || !original.Value.Span.SequenceEqual(proof.Encode())) + { + return null; + } + + CloudStateCheckpoint? members = await transaction.Checkpoints.GetAsync(MembersName(proof.ProofId), cancellationToken).ConfigureAwait(false); + return members is not null ? DecodeMembers(members.Value, proof.ProofId, proof.RootItemId) : + throw new InvalidDataException("The immutable directory preparation has no membership record."); + } + + internal static IReadOnlyList DecodeMembers(ReadOnlyMemory value, Guid evidenceId, Guid rootItemId) + { + if (value.Length is < 48 or > 64 * 1024 * 1024) + { + throw new InvalidDataException("The directory membership length is invalid."); + } + + try + { + using MemoryStream stream = new(value.ToArray(), writable: false); + using BinaryReader reader = new(stream); + if (reader.ReadInt32() != 1) + { + throw new NotSupportedException("The directory membership version is unsupported."); + } + + if (CloudDirectoryEvidenceCodec.ReadGuid(reader) != evidenceId) + { + throw new InvalidDataException("The directory membership belongs to another preparation."); + } + + int count = reader.ReadInt32(); + if (count <= 0 || count > (value.Length - 24) / 24) + { + throw new InvalidDataException("The directory membership count is invalid."); + } + + List members = new(count); + HashSet ids = []; + HashSet suffixes = new(StringComparer.OrdinalIgnoreCase); + bool hasRoot = false; + for (int index = 0; index < count; index++) + { + Guid id = CloudDirectoryEvidenceCodec.ReadGuid(reader); + CloudItemKind kind = (CloudItemKind)reader.ReadInt32(); + string suffix = CloudDirectoryEvidenceCodec.ReadText(reader); + if (suffix.Length != 0 && CloudDirectoryEvidenceCodec.CanonicalPath(suffix) != suffix) + { + throw new InvalidDataException("The directory membership suffix is not canonical."); + } + + if (id == Guid.Empty || !Enum.IsDefined(kind) || !ids.Add(id) || !suffixes.Add(suffix) || + (suffix.Length == 0 && (id != rootItemId || kind != CloudItemKind.Directory))) + { + throw new InvalidDataException("The directory membership is ambiguous."); + } + + hasRoot |= suffix.Length == 0; + members.Add(new(id, kind, suffix)); + } + + if (!hasRoot || stream.Position != stream.Length) + { + throw new InvalidDataException("The directory membership is incomplete."); + } + + return members.AsReadOnly(); + } + catch (Exception exception) when (exception is ArgumentException or EndOfStreamException) + { + throw new InvalidDataException("The directory membership is malformed.", exception); + } + } internal static async ValueTask GetScopeAsync(ICloudStateTransaction transaction, bool create, CancellationToken cancellationToken) @@ -121,6 +205,8 @@ internal static byte[] EncodeMembers(Guid evidenceId, Guid rootItemId, string so } } +internal sealed record CloudDirectoryMember(Guid ItemId, CloudItemKind Kind, string Suffix); + internal sealed record DirectoryNativeObservation(CloudLocalFileBinding Binding, byte[] Identity, string ActualPath) { internal bool Matches(DirectoryNativeObservation other) => Binding == other.Binding && diff --git a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs index 267235e..d109353 100644 --- a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs +++ b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs @@ -131,6 +131,45 @@ await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(BindingName(i } } + internal static async ValueTask ProjectPathsAsync(ICloudStateTransaction transaction, CloudDirectoryMoveProof proof, + IReadOnlyList members, IReadOnlyList current, CancellationToken cancellationToken) + { + Dictionary known = members.ToDictionary(member => member.ItemId); + foreach (CloudItemState item in current.Where(item => item.Kind == CloudItemKind.Directory && !item.IsTombstone)) + { + CloudDirectoryProvenance? provenance; + if (item.ItemId == proof.RootItemId) + { + provenance = new(Guid.NewGuid(), proof.StoreScope, item.ItemId, proof.DestinationRelativePath, + proof.ExpectedBinding, proof.ExpectedPlaceholderIdentity.ToArray()); + } + else + { + CloudStateCheckpoint? checkpoint = await transaction.Checkpoints.GetAsync(BindingName(item.ItemId), cancellationToken).ConfigureAwait(false); + if (checkpoint is null) + { + continue; + } + + provenance = Decode(checkpoint.Value); + string sourcePath = CloudDirectoryStateProjection.MemberPath(proof.SourceRelativePath, known[item.ItemId].Suffix); + string targetPath = CloudDirectoryStateProjection.MemberPath(proof.DestinationRelativePath, known[item.ItemId].Suffix); + if (provenance.RootItemId != item.ItemId || provenance.StoreScope != proof.StoreScope || + (!CloudDirectoryStateProjection.SamePath(provenance.RelativePath, sourcePath) && + !CloudDirectoryStateProjection.SamePath(provenance.RelativePath, targetPath))) + { + throw new CloudDirectoryProjectionConflictException("Historical directory provenance belongs to another namespace location."); + } + + provenance = provenance with { EvidenceId = Guid.NewGuid(), RelativePath = targetPath }; + } + + await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(BindingName(item.ItemId), provenance.Encode(), DateTimeOffset.UtcNow), + cancellationToken).ConfigureAwait(false); + await RefreshMembersAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); + } + } + private static async ValueTask RefreshMembersAsync(ICloudStateTransaction transaction, CloudDirectoryProvenance provenance, CancellationToken cancellationToken) { diff --git a/src/CfSharp/Persistence/CloudDirectoryStateProjection.cs b/src/CfSharp/Persistence/CloudDirectoryStateProjection.cs index 1861ccc..60b146c 100644 --- a/src/CfSharp/Persistence/CloudDirectoryStateProjection.cs +++ b/src/CfSharp/Persistence/CloudDirectoryStateProjection.cs @@ -4,6 +4,91 @@ namespace CfSharp; // performs native I/O, acknowledges journal rows, or infers native ownership from durable paths. internal static class CloudDirectoryStateProjection { + internal static async ValueTask ProjectVerifiedAsync(ICloudStateTransaction transaction, + CloudDirectoryMoveProof proof, IReadOnlyList members, + DateTimeOffset updatedAt, CancellationToken cancellationToken) + { + Dictionary known = members.ToDictionary(member => member.ItemId); + IReadOnlyList source = await transaction.Items.ListSubtreeAsync(proof.SourceRelativePath, cancellationToken).ConfigureAwait(false); + IReadOnlyList target = await transaction.Items.ListSubtreeAsync(proof.DestinationRelativePath, cancellationToken).ConfigureAwait(false); + foreach (CloudItemState item in source.Concat(target)) + { + if (!known.TryGetValue(item.ItemId, out CloudDirectoryMember? member) || !MatchesMember(item, member, proof)) + { + throw new CloudDirectoryProjectionConflictException("The current subtree contains unproven ownership or a changed member path."); + } + } + + List current = []; + List<(CloudItemState Item, string Path)> updates = []; + foreach (CloudDirectoryMember member in members) + { + CloudItemState? item = await transaction.Items.GetByItemIdAsync(member.ItemId, cancellationToken).ConfigureAwait(false); + if (item is null) + { + if (member.ItemId == proof.RootItemId) + { + throw new CloudDirectoryProjectionConflictException("The prepared directory root no longer has durable identity."); + } + + // Removal is not permission to resurrect an old prepared snapshot. + continue; + } + + if (!MatchesMember(item, member, proof) || (member.ItemId == proof.RootItemId && item.IsTombstone)) + { + throw new CloudDirectoryProjectionConflictException("A prepared member changed kind, ownership, or namespace location."); + } + + string path = MemberPath(proof.DestinationRelativePath, member.Suffix); + CloudItemState? collision = await transaction.Items.GetByRelativePathAsync(path, cancellationToken).ConfigureAwait(false); + if (collision is not null && collision.ItemId != item.ItemId) + { + throw new CloudDirectoryProjectionConflictException("An unrelated item owns a destination path."); + } + + current.Add(item); + if (!string.Equals(item.RelativePath, path, StringComparison.Ordinal)) + { + updates.Add((item, path)); + } + } + + CloudStateCheckpoint? receipt = await transaction.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId), cancellationToken).ConfigureAwait(false); + byte[] receiptValue = proof.Encode(); + if (receipt is not null && (!receipt.Value.Span.SequenceEqual(receiptValue) || updates.Count != 0)) + { + throw new CloudDirectoryProjectionConflictException("A historical completion receipt cannot regress later namespace state."); + } + + // Complete every preflight before the first write. Preserve the current revision, local + // ID and tombstone rather than restoring stale values from preparation. The journal is + // deliberately untouched: historical payload paths and ACK identities remain meaningful. + foreach ((CloudItemState item, string path) in updates) + { + await transaction.Items.UpsertAsync(new CloudItemState(item.ItemId, item.RemoteId, path, + item.Kind, item.RemoteRevision, item.LocalFileId, item.IsTombstone, updatedAt), cancellationToken).ConfigureAwait(false); + } + + if (receipt is null) + { + await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId), + receiptValue, updatedAt), cancellationToken).ConfigureAwait(false); + } + + return new(updates.Count, receipt is not null, current.AsReadOnly()); + } + + internal static string MemberPath(string root, string suffix) => suffix.Length == 0 ? root : Path.Combine(root, suffix); + + private static bool MatchesMember(CloudItemState item, CloudDirectoryMember member, CloudDirectoryMoveProof proof) => + item.Kind == member.Kind && + (SamePath(item.RelativePath, MemberPath(proof.SourceRelativePath, member.Suffix)) || + SamePath(item.RelativePath, MemberPath(proof.DestinationRelativePath, member.Suffix))); + + internal static bool SamePath(string first, string second) => + string.Equals(first.Replace('/', '\\'), second.Replace('/', '\\'), StringComparison.OrdinalIgnoreCase); + internal static async ValueTask ProjectAsync(ICloudStateTransaction transaction, IReadOnlyList sourceEntries, string sourceRelativePath, string destinationRelativePath, DateTimeOffset updatedAt, CancellationToken cancellationToken) @@ -36,3 +121,7 @@ internal static string MapPath(string relativePath, string sourceRelativePath, s return suffix.Length == 0 ? destinationRelativePath : Path.Combine(destinationRelativePath, suffix); } } + +internal sealed record VerifiedDirectoryProjection(int UpdatedCount, bool AlreadyCompleted, IReadOnlyList CurrentItems); + +internal sealed class CloudDirectoryProjectionConflictException(string message) : InvalidOperationException(message); diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveReconciliationTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveReconciliationTests.cs new file mode 100644 index 0000000..b259b71 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveReconciliationTests.cs @@ -0,0 +1,202 @@ +using System.ComponentModel; +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReconciliationCancellationReportsTheActualNativeAndCommitStage(bool duringProjection) + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + using CancellationTokenSource cancellation = new(); + if (duringProjection) + { + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Moved")); + root.Faults.BeforeCommit = cancellation.Cancel; + } + else + { + cancellation.Cancel(); + } + + CloudDirectoryMoveReconciliationResult canceled = await source.ReconcileMoveAsync(proof, cancellation.Token); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Canceled, canceled.Outcome); + Assert.Equal(duringProjection ? CloudDirectoryMoveReconciliationStage.DurableProjection : + CloudDirectoryMoveReconciliationStage.Acquisition, canceled.Stage); + Assert.Equal(duringProjection, canceled.NativeMoveObserved); + Assert.False(canceled.DurableProjectionCommitted); + Assert.IsAssignableFrom(canceled.Error); + Assert.Null(canceled.NativeHResult); + root.Faults.BeforeCommit = null; + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal("Docs", (await verify.Items.GetByItemIdAsync(proof.RootItemId))!.RelativePath); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + } + + [Fact] + public async Task ReconcileProjectsCurrentFieldsPartialRootAndReceiptWithoutTouchingPendingJournal() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudItemState child; + CloudOperationJournalEntry pending; + await using (ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync()) + { + child = (await transaction.Items.GetByRelativePathAsync("Docs\\child.txt"))!; + pending = await transaction.Operations.EnqueueAsync(new(Guid.NewGuid(), CloudStateOperationKind.ContentUpdate, + child.ItemId, new LocalChangePayload("Docs\\child.txt", null, false, DateTimeOffset.UtcNow).Encode(), DateTimeOffset.UtcNow)); + await transaction.CommitAsync(); + } + + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + CloudDirectoryMoveReconciliationResult notMoved = await source.ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.NotMoved, notMoved.Outcome); + Assert.False(notMoved.NativeMoveObserved); + Assert.False(notMoved.DurableProjectionCommitted); + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Moved")); + await using (ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync()) + { + CloudItemState originalRoot = (await transaction.Items.GetByItemIdAsync(proof.RootItemId))!; + await transaction.Items.UpsertAsync(new(originalRoot.ItemId, originalRoot.RemoteId, "Moved", originalRoot.Kind, + "root-current", 55, false, DateTimeOffset.UtcNow)); + await transaction.Items.UpsertAsync(new(child.ItemId, child.RemoteId, child.RelativePath, child.Kind, + "child-current", 66, false, DateTimeOffset.UtcNow)); + await transaction.CommitAsync(); + } + + CloudDirectoryMoveReconciliationResult projected = await source.ReconcileMoveAsync(CloudDirectoryMoveProof.Decode(proof.Encode())); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, projected.Outcome); + Assert.True(projected.NativeMoveObserved); + Assert.True(projected.DurableProjectionCommitted); + Assert.False(projected.RequiresFullRescan); + Assert.Equal(1, projected.DurableStateEntriesUpdated); + Assert.Null(projected.Error); + Assert.Null(projected.NativeHResult); + CloudDirectoryMoveReconciliationResult duplicate = await source.ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.AlreadyProjected, duplicate.Outcome); + Assert.True(duplicate.DurableProjectionCommitted); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + CloudItemState movedChild = (await verify.Items.GetByItemIdAsync(child.ItemId))!; + Assert.Equal("Moved\\child.txt", movedChild.RelativePath); + Assert.Equal("child-current", movedChild.RemoteRevision); + Assert.Equal(66, movedChild.LocalFileId); + CloudItemState movedRoot = (await verify.Items.GetByItemIdAsync(proof.RootItemId))!; + Assert.Equal("root-current", movedRoot.RemoteRevision); + Assert.Equal(55, movedRoot.LocalFileId); + CloudOperationJournalEntry retained = (await verify.Operations.GetAsync(pending.OperationId))!; + Assert.Equal(pending.Sequence, retained.Sequence); + Assert.Equal(pending.Payload.ToArray(), retained.Payload.ToArray()); + Assert.Equal(pending.CreatedAt, retained.CreatedAt); + Assert.Equal(proof.Encode(), (await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId)))!.Value.ToArray()); + Assert.Equal("Moved", CloudDirectoryProvenance.Decode((await verify.Checkpoints.GetAsync( + CloudDirectoryProvenance.BindingName(proof.RootItemId)))!.Value).RelativePath); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProjectionFailureRetainsNativeFactsAndOriginalErrorWithoutFakeNativeHResult(bool win32StorageError) + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Moved")); + Exception error = win32StorageError ? new Win32Exception(5, "Storage commit fault.") : new IOException("Storage commit fault."); + root.Faults.BeforeCommit = () => throw error; + CloudDirectoryMoveReconciliationResult failed = await source.ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.NativeObservedProjectionPending, failed.Outcome); + Assert.Equal(CloudDirectoryMoveReconciliationStage.DurableProjection, failed.Stage); + Assert.True(failed.NativeMoveObserved); + Assert.False(failed.DurableProjectionCommitted); + Assert.Same(error, failed.Error); + Assert.Null(failed.NativeHResult); + root.Faults.BeforeCommit = null; + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + Assert.Equal("Docs", (await verify.Items.GetByItemIdAsync(proof.RootItemId))!.RelativePath); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + } + + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, (await source.ReconcileMoveAsync(proof)).Outcome); + Assert.False(Directory.Exists(source.FullPath)); + } + + [Fact] + public async Task ForeignForgedAndUnrelatedTargetStateFailClosedWithoutDeletingRows() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + await using DirectoryMoveTestRoot foreign = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.NotApplicable, + (await foreign.FileSystem.GetDirectory("Docs").ReconcileMoveAsync(proof)).Outcome); + CloudDirectoryMoveProof forged = new(Guid.NewGuid(), proof.StoreScope, proof.SourceRelativePath, proof.DestinationRelativePath, + proof.RootItemId, proof.ExpectedBinding, proof.ExpectedPlaceholderIdentity.Span); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.NotApplicable, + (await source.ReconcileMoveAsync(CloudDirectoryMoveProof.Decode(forged.Encode()))).Outcome); + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Moved")); + Guid unrelated = Guid.NewGuid(); + await using (ICloudStateTransaction seed = await root.Store.BeginTransactionAsync()) + { + await seed.Items.UpsertAsync(new(unrelated, "unrelated", "Moved\\unrelated", CloudItemKind.File, + "untouched", 91, true, DateTimeOffset.UtcNow)); + await seed.CommitAsync(); + } + + CloudDirectoryMoveReconciliationResult conflict = await source.ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Conflict, conflict.Outcome); + Assert.True(conflict.NativeMoveObserved); + Assert.False(conflict.DurableProjectionCommitted); + Assert.True(conflict.RequiresFullRescan); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.NotNull(await verify.Items.GetByItemIdAsync(unrelated)); + Assert.Equal("Docs", (await verify.Items.GetByItemIdAsync(proof.RootItemId))!.RelativePath); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + } + + [Fact] + public async Task CaseOnlyRecoveryUsesActualNamespaceSpellingAndNoSecondNativeMove() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("DOCS"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "docs"); + Assert.Equal("Docs", proof.SourceRelativePath); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.NotMoved, (await source.ReconcileMoveAsync(proof)).Outcome); + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "docs")); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, (await source.ReconcileMoveAsync(proof)).Outcome); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal("docs", (await verify.Items.GetByItemIdAsync(proof.RootItemId))!.RelativePath); + Assert.Equal("docs\\child.txt", (await verify.Items.GetByRelativePathAsync("docs\\child.txt"))!.RelativePath); + } +} diff --git a/tests/CfSharp.Tests/Persistence/VerifiedDirectoryProjectionTests.cs b/tests/CfSharp.Tests/Persistence/VerifiedDirectoryProjectionTests.cs new file mode 100644 index 0000000..c5d15a9 --- /dev/null +++ b/tests/CfSharp.Tests/Persistence/VerifiedDirectoryProjectionTests.cs @@ -0,0 +1,99 @@ +namespace CfSharp.Tests.Persistence; + +public sealed class VerifiedDirectoryProjectionTests +{ + [Fact] + public async Task CurrentTombstonesAndRevisionsSurvivePartialProjectionAndReceiptCannotRegressLaterPaths() + { + await using ICloudStateStore store = await InMemoryCloudStateStoreContractTests.CreateFactoryForTesting() + .OpenAsync(new CloudStateStoreContext(Path.GetTempPath())); + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Create("root"); + CloudDirectoryMoveProof proof = new(Guid.NewGuid(), Guid.NewGuid(), "Docs", "Moved", identity.ItemId, + new CloudLocalFileBinding(1, Guid.NewGuid(), Guid.NewGuid()), identity.Encode()); + CloudItemState originalRoot = new(identity.ItemId, "root", "Docs", CloudItemKind.Directory, "old", 1, false, DateTimeOffset.UtcNow); + CloudItemState originalChild = new(Guid.NewGuid(), "child", "Docs\\child", CloudItemKind.File, "old", 2, false, DateTimeOffset.UtcNow); + CloudItemState sibling = new(Guid.NewGuid(), "sibling", "Docs2\\child", CloudItemKind.File, "sibling", 3, false, DateTimeOffset.UtcNow); + IReadOnlyList members = CloudDirectoryMoveEvidence.DecodeMembers( + CloudDirectoryMoveEvidence.EncodeMembers(proof, [originalRoot, originalChild]), proof.ProofId, proof.RootItemId); + await using (ICloudStateTransaction seed = await store.BeginTransactionAsync()) + { + await seed.Items.UpsertAsync(new(originalRoot.ItemId, originalRoot.RemoteId, "Moved", originalRoot.Kind, + "root-current", 11, false, DateTimeOffset.UtcNow)); + await seed.Items.UpsertAsync(new(originalChild.ItemId, originalChild.RemoteId, originalChild.RelativePath, + originalChild.Kind, "child-current", 22, true, DateTimeOffset.UtcNow)); + await seed.Items.UpsertAsync(sibling); + await seed.CommitAsync(); + } + + await using (ICloudStateTransaction rollback = await store.BeginTransactionAsync()) + { + VerifiedDirectoryProjection projection = await CloudDirectoryStateProjection.ProjectVerifiedAsync(rollback, proof, + members, DateTimeOffset.UtcNow, default); + Assert.Equal(1, projection.UpdatedCount); + Assert.False(projection.AlreadyCompleted); + } + + await using (ICloudStateTransaction project = await store.BeginTransactionAsync()) + { + Assert.Equal("Docs\\child", (await project.Items.GetByItemIdAsync(originalChild.ItemId))!.RelativePath); + Assert.Null(await project.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + await CloudDirectoryStateProjection.ProjectVerifiedAsync(project, proof, members, DateTimeOffset.UtcNow, default); + await project.CommitAsync(); + } + + await using (ICloudStateTransaction duplicate = await store.BeginTransactionAsync()) + { + VerifiedDirectoryProjection projection = await CloudDirectoryStateProjection.ProjectVerifiedAsync(duplicate, proof, + members, DateTimeOffset.UtcNow, default); + Assert.True(projection.AlreadyCompleted); + Assert.Equal(0, projection.UpdatedCount); + CloudItemState child = (await duplicate.Items.GetByItemIdAsync(originalChild.ItemId))!; + Assert.Equal("Moved\\child", child.RelativePath); + Assert.Equal("child-current", child.RemoteRevision); + Assert.Equal(22, child.LocalFileId); + Assert.True(child.IsTombstone); + Assert.Equal(sibling.RelativePath, (await duplicate.Items.GetByItemIdAsync(sibling.ItemId))!.RelativePath); + await duplicate.Items.UpsertAsync(new(originalRoot.ItemId, originalRoot.RemoteId, "Later", originalRoot.Kind, + "later", 33, false, DateTimeOffset.UtcNow)); + await duplicate.CommitAsync(); + } + + await using ICloudStateTransaction late = await store.BeginTransactionAsync(); + await Assert.ThrowsAsync(() => CloudDirectoryStateProjection.ProjectVerifiedAsync( + late, proof, members, DateTimeOffset.UtcNow, default).AsTask()); + Assert.Equal("Later", (await late.Items.GetByItemIdAsync(proof.RootItemId))!.RelativePath); + Assert.Equal(proof.Encode(), (await late.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId)))!.Value.ToArray()); + } + + [Theory] + [InlineData("Docs\\unproven")] + [InlineData("Moved\\unproven")] + public async Task UnprovenMembershipFailsBeforeAnyWriteEvenWhenCallerCommits(string unprovenPath) + { + await using ICloudStateStore store = await InMemoryCloudStateStoreContractTests.CreateFactoryForTesting() + .OpenAsync(new CloudStateStoreContext(Path.GetTempPath())); + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Create("root"); + CloudDirectoryMoveProof proof = new(Guid.NewGuid(), Guid.NewGuid(), "Docs", "Moved", identity.ItemId, + new CloudLocalFileBinding(1, Guid.NewGuid(), Guid.NewGuid()), identity.Encode()); + CloudItemState root = new(identity.ItemId, "root", "Docs", CloudItemKind.Directory, "old", 1, false, DateTimeOffset.UtcNow); + Guid unrelated = Guid.NewGuid(); + await using (ICloudStateTransaction seed = await store.BeginTransactionAsync()) + { + await seed.Items.UpsertAsync(root); + await seed.Items.UpsertAsync(new(unrelated, "unproven", unprovenPath, CloudItemKind.File, "unproven", 2, true, DateTimeOffset.UtcNow)); + await seed.CommitAsync(); + } + + await using (ICloudStateTransaction transaction = await store.BeginTransactionAsync()) + { + await Assert.ThrowsAsync(() => CloudDirectoryStateProjection.ProjectVerifiedAsync( + transaction, proof, [new(root.ItemId, root.Kind, "")], DateTimeOffset.UtcNow, default).AsTask()); + await transaction.CommitAsync(); + } + + await using ICloudStateTransaction verify = await store.BeginTransactionAsync(); + Assert.Equal("Docs", (await verify.Items.GetByItemIdAsync(root.ItemId))!.RelativePath); + Assert.Equal(unprovenPath, (await verify.Items.GetByItemIdAsync(unrelated))!.RelativePath); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + } +} From d50e096ffb73e7364efc5847a5b7846188cf2f80 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:46:01 +0800 Subject: [PATCH 12/40] fix(items): verify directory move replay with retained proofs --- docs/placeholders.md | 13 ++ eng/api-baseline.json | 4 + ...loudFileSystem.DirectoryMovePreparation.cs | 5 + .../CloudFileSystem.NamespaceOperations.cs | 113 +++++++++++ .../Items/CloudDirectory.MovePreparation.cs | 5 +- .../Items/CloudItem.NamespaceOperations.cs | 9 +- .../Items/CloudItemOperationResults.cs | 29 ++- .../Persistence/CloudDirectoryMoveEvidence.cs | 114 ++++++++++- .../Persistence/CloudDirectoryProvenance.cs | 2 +- .../Placeholders/CloudPlaceholderOptions.cs | 17 ++ .../CloudDirectoryMoveReplayTests.cs | 183 ++++++++++++++++++ 11 files changed, 481 insertions(+), 13 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveReplayTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index 0d16ff9..6b7c091 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -37,6 +37,19 @@ changes or identity writes. Validation around commit detects changed known desce ordinary children without native placeholder identity require a rescan before dispatch. A detected post-commit race reports the committed fact and rescan requirement rather than claiming rollback. +`MoveToAsync` prepares supported managed directories before its native move. Retrying the original +source/destination after a crash uses the retained indexed preparation and performs only verified +projection. To select a specific externally prepared proof, pass `new CloudMoveOptions(proof)`; +it must name that exact target. Missing-source directory retries without evidence fail closed. +Normal ordinary-directory moves and the existing boolean file-replacement options remain available. +Directory move results and coordination exceptions expose `DirectoryReconciliation`, preserving a +post-commit race or rescan requirement without inventing a failed commit. The bounded intent index +may select a newer preparation but never replaces previous immutable proofs or receipts. +When no explicit preparation exists, recovery may promote the library's retained pre-rename +directory binding and known membership into an independent immutable preparation. It never +manufactures historical evidence from a current target. Legacy directories without either source +of provenance still require explicit preparation while at the original source. + Placeholder operations use immutable, kind-specific specifications rather than exposing native unions and flag combinations directly to application code. diff --git a/eng/api-baseline.json b/eng/api-baseline.json index fd63126..5e1e531 100644 --- a/eng/api-baseline.json +++ b/eng/api-baseline.json @@ -990,6 +990,7 @@ "property System.String RelativePath()", "type CfSharp.CloudItemCoordinationException", "property System.String DestinationPath()", + "property CfSharp.CloudDirectoryMoveReconciliationResult DirectoryReconciliation()", "property System.String Operation()", "property System.Nullable\u003CSystem.Int64\u003E OperationUsn()", "property CfSharp.CloudRecursiveOperationResult PartialResult()", @@ -1032,6 +1033,7 @@ "property CfSharp.CloudItemLeaseOptions ReadOnly()", "type CfSharp.CloudItemMoveResult", "property System.String DestinationPath()", + "property CfSharp.CloudDirectoryMoveReconciliationResult DirectoryReconciliation()", "property System.Int32 DurableStateEntriesUpdated()", "property CfSharp.CloudItem Item()", "property CfSharp.CloudItemSnapshot Snapshot()", @@ -1153,8 +1155,10 @@ "field CfSharp.CloudMetadataWriteMode PreserveUnspecified", "field System.Int32 value__", "type CfSharp.CloudMoveOptions", + "ctor CfSharp.CloudMoveOptions(CfSharp.CloudDirectoryMoveProof directoryMoveProof)", "ctor CfSharp.CloudMoveOptions(System.Boolean replaceExisting)", "property CfSharp.CloudMoveOptions Default()", + "property CfSharp.CloudDirectoryMoveProof DirectoryMoveProof()", "property System.Boolean ReplaceExisting()", "type CfSharp.CloudOperationJournalEntry", "ctor CfSharp.CloudOperationJournalEntry(System.Guid operationId,CfSharp.CloudStateOperationKind kind,System.Nullable\u003CSystem.Guid\u003E itemId,System.ReadOnlySpan\u003CSystem.Byte\u003E payload,System.DateTimeOffset createdAt,System.Int32 attemptCount,System.Nullable\u003CSystem.DateTimeOffset\u003E retryAfter,System.Int64 sequence)", diff --git a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs index faf5f89..0e1eb40 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs @@ -57,6 +57,11 @@ internal async ValueTask PrepareDirectoryMoveAsync(Clou root.ItemId, captured.Binding, captured.Identity); IReadOnlyList members = await transaction.Items.ListSubtreeAsync(source.RelativePath, cancellationToken).ConfigureAwait(false); await CloudDirectoryMoveEvidence.PrepareAsync(transaction, proof, members, cancellationToken).ConfigureAwait(false); + // The bounded pair index selects the latest preparation for legacy MoveTo retry. It may + // change without overwriting any immutable proof, member manifest, or completion receipt. + await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint( + CloudDirectoryMoveEvidence.IntentName(proof.SourceRelativePath, proof.DestinationRelativePath), + proof.Encode(), DateTimeOffset.UtcNow), cancellationToken).ConfigureAwait(false); if (!captured.Matches(CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), SyncRootPath))) { throw new InvalidOperationException("The native directory changed during preparation."); diff --git a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs index d4b3a54..1d0b868 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs @@ -36,6 +36,13 @@ internal async ValueTask MoveAsync( ? validatedName : Path.Combine(destination.RelativePath, validatedName); CloudItem movedItem = CreateItemReference(destinationRelativePath, item.Kind); + CloudDirectoryMoveProof? directoryProof = options.DirectoryMoveProof; + if (directoryProof is not null && (item.Kind != CloudItemKind.Directory || + !CloudDirectoryStateProjection.SamePath(item.RelativePath, directoryProof.SourceRelativePath) || + movedItem.RelativePath != directoryProof.DestinationRelativePath)) + { + throw new ArgumentException("The directory proof must identify this source and exact intended destination.", nameof(options)); + } if (item.Kind is CloudItemKind.Directory && IsStrictDescendant(movedItem.FullPath, item.FullPath)) { @@ -52,6 +59,10 @@ internal async ValueTask MoveAsync( using CloudFileSystemOperationLease operation = await AcquireOperationAsync( [sourceScope, destinationScope], cancellationToken).ConfigureAwait(false); + if (operation.Covers([sourceScope, destinationScope])) + { + operation.EstablishContext(); + } LocalCloudItemInspection sourceState = CloudItemInspector.Inspect(item.FullPath, item.Kind); LocalCloudItemInspection destinationDirectoryState = CloudItemInspector.Inspect( @@ -63,6 +74,34 @@ internal async ValueTask MoveAsync( $"The destination directory does not exist: '{destination.FullPath}'."); } + if (item is CloudDirectory directory && (directoryProof is not null || !sourceState.Exists)) + { + directoryProof ??= await CloudDirectoryMoveEvidence.ReadIntentAsync(operation.StateStore, + item.RelativePath, movedItem.RelativePath, cancellationToken).ConfigureAwait(false); + if (directoryProof is null) + { + throw new FileNotFoundException("Directory recovery requires original durable pre-move evidence.", item.FullPath); + } + + CloudDirectoryMoveReconciliationResult recovery = await ReconcileDirectoryMoveAsync(directory, + directoryProof, cancellationToken).ConfigureAwait(false); + if (IsCompletedDirectoryRecovery(recovery)) + { + return await CreateRecoveredMoveResultAsync(item, movedItem, operation.StateStore, recovery).ConfigureAwait(false); + } + + if (recovery.Outcome != CloudDirectoryMoveReconciliationOutcome.NotMoved || recovery.RequiresFullRescan) + { + ThrowDirectoryRecoveryFailure(item, movedItem, recovery, nativeMoveIssued: false, cancellationToken); + } + + sourceState = CloudItemInspector.Inspect(item.FullPath, item.Kind); + if (!sourceState.Exists) + { + throw new FileNotFoundException("The directory changed during move validation; retry its original proof.", item.FullPath); + } + } + MoveStatePlan statePlan = await ReadMoveStatePlanAsync( operation.StateStore, item, @@ -136,6 +175,20 @@ await PersistMovedStateAsync( } cancellationToken.ThrowIfCancellationRequested(); + if (item is CloudDirectory managedDirectory && directoryProof is null && HasManagedDirectoryRoot(sourceState, statePlan.SourceEntries, item.RelativePath)) + { + try + { + directoryProof = await PrepareDirectoryMoveAsync(managedDirectory, destination, + validatedName, cancellationToken).ConfigureAwait(false); + } + catch (CloudDirectoryEvidenceUnavailableException) + { + // Additive recovery evidence cannot remove ordinary native move availability + // on storage without complete IDs or beyond the bounded preparation capacity. + } + } + try { if (item.Kind is CloudItemKind.File) @@ -152,6 +205,18 @@ await PersistMovedStateAsync( throw CloudFilesException.FromException("CloudItem.Move", item.FullPath, exception); } + if (item is CloudDirectory movedDirectory && directoryProof is not null) + { + CloudDirectoryMoveReconciliationResult recovery = await ReconcileDirectoryMoveAsync(movedDirectory, + directoryProof, CancellationToken.None).ConfigureAwait(false); + if (!IsCompletedDirectoryRecovery(recovery)) + { + ThrowDirectoryRecoveryFailure(item, movedItem, recovery, nativeMoveIssued: true, CancellationToken.None); + } + + return await CreateRecoveredMoveResultAsync(item, movedItem, operation.StateStore, recovery).ConfigureAwait(false); + } + try { await PersistMovedStateAsync( @@ -183,6 +248,54 @@ await PersistMovedStateAsync( statePlan.SourceEntries.Count); } + private static bool HasManagedDirectoryRoot(LocalCloudItemInspection native, + IReadOnlyList entries, string relativePath) + { + if (!native.PlaceholderState.HasFlag(CloudPlaceholderState.Placeholder)) + { + return false; + } + + try + { + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Decode(native.PlaceholderIdentity); + return entries.Any(entry => entry.ItemId == identity.ItemId && entry.RemoteId == identity.RemoteId && + entry.Kind == CloudItemKind.Directory && !entry.IsTombstone && + CloudDirectoryStateProjection.SamePath(entry.RelativePath, relativePath)); + } + catch (Exception exception) when (exception is InvalidDataException or NotSupportedException) + { + return false; + } + } + + private static bool IsCompletedDirectoryRecovery(CloudDirectoryMoveReconciliationResult recovery) => + recovery.Outcome is CloudDirectoryMoveReconciliationOutcome.Projected or CloudDirectoryMoveReconciliationOutcome.AlreadyProjected && + recovery.DurableProjectionCommitted; + + private static async ValueTask CreateRecoveredMoveResultAsync(CloudItem source, + CloudItem target, ICloudStateStore store, CloudDirectoryMoveReconciliationResult recovery) => + new(source.FullPath, target.FullPath, target, await InspectCoreAsync(target, store, CancellationToken.None).ConfigureAwait(false), + recovery.DurableStateEntriesUpdated, recovery); + + private static void ThrowDirectoryRecoveryFailure(CloudItem source, CloudItem target, + CloudDirectoryMoveReconciliationResult recovery, bool nativeMoveIssued, CancellationToken cancellationToken) + { + Exception error = recovery.Error ?? new InvalidOperationException( + $"The directory proof cannot coordinate this move ({recovery.Outcome}); reconcile the namespace before retrying."); + if (nativeMoveIssued || recovery.NativeMoveObserved) + { + throw new CloudItemCoordinationException("CloudItem.Move", source.FullPath, target.FullPath, null, error, recovery); + } + + if (recovery.Outcome == CloudDirectoryMoveReconciliationOutcome.Canceled) + { + throw new OperationCanceledException("Directory move validation was canceled.", error, cancellationToken); + } + + throw new InvalidOperationException("The directory move proof could not be verified.", error); + } + internal async ValueTask DeleteAsync( CloudItem item, CancellationToken cancellationToken) diff --git a/src/CfSharp/Items/CloudDirectory.MovePreparation.cs b/src/CfSharp/Items/CloudDirectory.MovePreparation.cs index f018b7c..dd45016 100644 --- a/src/CfSharp/Items/CloudDirectory.MovePreparation.cs +++ b/src/CfSharp/Items/CloudDirectory.MovePreparation.cs @@ -19,7 +19,10 @@ public sealed partial class CloudDirectory /// /// The destination belongs to another file system or the name is invalid. /// The root, destination, or durable/native identity is unsuitable. - /// Storage cannot supply the complete native binding. + /// + /// Storage cannot supply the complete native binding, known subtree metadata exceeds 64 MiB, + /// or durable recovery records use an unsupported protocol version. + /// /// Native metadata, path protection, or durable preparation failed. /// Preparation was canceled; no proof is returned. public ValueTask PrepareMoveAsync(CloudDirectory destination, string name, diff --git a/src/CfSharp/Items/CloudItem.NamespaceOperations.cs b/src/CfSharp/Items/CloudItem.NamespaceOperations.cs index 292ca32..f5d5738 100644 --- a/src/CfSharp/Items/CloudItem.NamespaceOperations.cs +++ b/src/CfSharp/Items/CloudItem.NamespaceOperations.cs @@ -5,13 +5,17 @@ public abstract partial class CloudItem /// Moves or renames this item within the owning sync root. /// Existing destination directory owned by the same file system. /// One valid destination child name. - /// Collision behavior, or null for no replacement. + /// Collision behavior or an original directory proof, or null for no replacement. /// /// Token observed before the synchronous file-system move and during durable-state work. /// /// /// A new immutable reference and snapshot at the destination. This reference remains bound to /// the old path and is not mutated. + /// Managed directory results also expose + /// and its rescan requirement. Supported managed directories prepare durable evidence before + /// moving; a retry uses that retained evidence or the explicitly supplied proof and never moves + /// an absent source again. An absent directory without historical evidence fails closed. /// /// /// The destination belongs to another file system, the name is invalid, or replacement was @@ -23,7 +27,8 @@ public abstract partial class CloudItem /// /// Windows rejects the namespace move. /// - /// Windows moved the item but durable subtree paths could not be committed. + /// Windows moved the item but durable subtree paths or native completion validation failed. + /// Directory recovery facts preserve whether projection actually committed. /// /// /// Cancellation was observed before the synchronous move or during pre-move state access. diff --git a/src/CfSharp/Items/CloudItemOperationResults.cs b/src/CfSharp/Items/CloudItemOperationResults.cs index fd4e866..1f79073 100644 --- a/src/CfSharp/Items/CloudItemOperationResults.cs +++ b/src/CfSharp/Items/CloudItemOperationResults.cs @@ -138,7 +138,8 @@ internal CloudItemMoveResult( string destinationPath, CloudItem item, CloudItemSnapshot snapshot, - int durableStateEntriesUpdated) + int durableStateEntriesUpdated, + CloudDirectoryMoveReconciliationResult? directoryReconciliation = null) { ArgumentException.ThrowIfNullOrWhiteSpace(sourcePath); ArgumentException.ThrowIfNullOrWhiteSpace(destinationPath); @@ -150,6 +151,7 @@ internal CloudItemMoveResult( Item = item; Snapshot = snapshot; DurableStateEntriesUpdated = durableStateEntriesUpdated; + DirectoryReconciliation = directoryReconciliation; } /// Gets the normalized absolute source path used by the operation. @@ -166,6 +168,13 @@ internal CloudItemMoveResult( /// Gets the number of durable item paths updated in the committed transaction. public int DurableStateEntriesUpdated { get; } + + /// + /// Gets verified directory coordination facts, including any required rescan, or null for a + /// file, ordinary directory, unchanged path, or storage without supported recovery evidence. + /// The value owns no protection lifetime and does not acknowledge pending operations. + /// + public CloudDirectoryMoveReconciliationResult? DirectoryReconciliation { get; } } /// Describes a completed file or empty-directory deletion. @@ -203,8 +212,8 @@ internal CloudItemDeleteResult( } /// -/// Reports that a file-system mutation completed but its corresponding durable-state change did -/// not commit. +/// Reports that a file-system mutation completed but durable coordination or native completion +/// validation failed. /// /// /// Windows and a configured state store cannot share one physical transaction. The exception @@ -227,9 +236,12 @@ internal CloudItemCoordinationException( string path, string? destinationPath, long? operationUsn, - Exception innerException) + Exception innerException, + CloudDirectoryMoveReconciliationResult? directoryReconciliation = null) : base( - destinationPath is null + directoryReconciliation?.DurableProjectionCommitted == true + ? $"File-system operation '{operation}' completed for '{path}' and committed its paths, but native completion requires reconciliation." + : destinationPath is null ? $"File-system operation '{operation}' completed for '{path}', but its durable state could not be committed." : $"File-system operation '{operation}' moved '{path}' to '{destinationPath}', but its durable state could not be committed.", innerException) @@ -240,6 +252,7 @@ destinationPath is null Path = path; DestinationPath = destinationPath; OperationUsn = operationUsn; + DirectoryReconciliation = directoryReconciliation; } internal CloudItemCoordinationException( @@ -271,4 +284,10 @@ internal CloudItemCoordinationException( /// for a non-recursive operation. /// public CloudRecursiveOperationResult? PartialResult { get; } + + /// + /// Gets directory recovery facts, including a real commit followed by failed native validation, + /// or null for other coordination failures. The original underlying failure remains the inner exception. + /// + public CloudDirectoryMoveReconciliationResult? DirectoryReconciliation { get; } } diff --git a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs index 64c2d4d..7e9ae21 100644 --- a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs +++ b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs @@ -1,4 +1,5 @@ using System.Runtime.Versioning; +using System.Security.Cryptography; using System.Text; using CfSharp.Native; @@ -12,6 +13,91 @@ internal static class CloudDirectoryMoveEvidence internal static string ProofName(Guid id) => $"cfsharp/namespace/preparations/{id:N}/proof"; internal static string MembersName(Guid id) => $"cfsharp/namespace/preparations/{id:N}/members"; internal static string ReceiptName(Guid id) => $"cfsharp/namespace/preparations/{id:N}/receipt"; + internal static string IntentName(string source, string target) => "cfsharp/namespace/intents/" + + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes( + CloudDirectoryEvidenceCodec.CanonicalPath(source).ToUpperInvariant() + "\0" + CloudDirectoryEvidenceCodec.CanonicalPath(target)))); + + internal static async ValueTask ReadIntentAsync(ICloudStateStore store, + string source, string target, CancellationToken cancellationToken) + { + await using ICloudStateTransaction transaction = await store.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + CloudDirectoryMoveProof? proof = await GetOrPrepareRetainedIntentAsync(transaction, source, target, cancellationToken).ConfigureAwait(false); + if (proof is not null) + { + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + + return proof; + } + + internal static async ValueTask GetOrPrepareRetainedIntentAsync(ICloudStateTransaction transaction, + string source, string target, CancellationToken cancellationToken) + { + CloudStateCheckpoint? checkpoint = await transaction.Checkpoints.GetAsync(IntentName(source, target), cancellationToken).ConfigureAwait(false); + if (checkpoint is not null) + { + CloudDirectoryMoveProof indexed = CloudDirectoryMoveProof.Decode(checkpoint.Value.Span); + return CloudDirectoryStateProjection.SamePath(source, indexed.SourceRelativePath) && target == indexed.DestinationRelativePath + ? indexed : throw new InvalidDataException("The directory intent index identifies another move."); + } + + Guid scope = await GetScopeAsync(transaction, false, cancellationToken).ConfigureAwait(false); + if (scope == Guid.Empty) + { + return null; + } + + CloudItemState? from = await transaction.Items.GetByRelativePathAsync(source, cancellationToken).ConfigureAwait(false); + CloudItemState? to = await transaction.Items.GetByRelativePathAsync(target, cancellationToken).ConfigureAwait(false); + foreach (CloudItemState item in new[] { from, to }.OfType().DistinctBy(item => item.ItemId)) + { + if (item.Kind != CloudItemKind.Directory || item.IsTombstone) + { + continue; + } + + CloudStateCheckpoint? retained = await transaction.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(item.ItemId), cancellationToken).ConfigureAwait(false); + if (retained is null) + { + continue; + } + + CloudDirectoryProvenance provenance = CloudDirectoryProvenance.Decode(retained.Value); + if (provenance.StoreScope != scope || provenance.RootItemId != item.ItemId || + !CloudDirectoryStateProjection.SamePath(source, provenance.RelativePath)) + { + continue; + } + + CloudStateCheckpoint? manifest = await transaction.Checkpoints.GetAsync(CloudDirectoryProvenance.MembersName(item.ItemId), cancellationToken).ConfigureAwait(false); + if (manifest is null) + { + throw new InvalidDataException("Retained directory provenance has no historical membership."); + } + + _ = DecodeMembers(manifest.Value, provenance.EvidenceId, provenance.RootItemId); + CloudDirectoryMoveProof prepared = new(Guid.NewGuid(), scope, provenance.RelativePath, target, + provenance.RootItemId, provenance.Binding, provenance.Identity); + // Promote library-captured historical evidence, never a binding read from today's + // target. Keep the private suffixes unchanged; only rebind the manifest owner GUID + // to this independent preparation (the same opaque 16-byte Guid encoding). + byte[] members = manifest.Value.ToArray(); + prepared.ProofId.TryWriteBytes(members.AsSpan(4, 16)); + DateTimeOffset now = DateTimeOffset.UtcNow; + if (await transaction.Checkpoints.GetAsync(ProofName(prepared.ProofId), cancellationToken).ConfigureAwait(false) is not null || + await transaction.Checkpoints.GetAsync(MembersName(prepared.ProofId), cancellationToken).ConfigureAwait(false) is not null) + { + throw new InvalidOperationException("A directory preparation is immutable."); + } + + await transaction.Checkpoints.UpsertAsync(new(ProofName(prepared.ProofId), prepared.Encode(), now), cancellationToken).ConfigureAwait(false); + await transaction.Checkpoints.UpsertAsync(new(MembersName(prepared.ProofId), members, now), cancellationToken).ConfigureAwait(false); + await transaction.Checkpoints.UpsertAsync(new(IntentName(source, target), prepared.Encode(), now), cancellationToken).ConfigureAwait(false); + return prepared; + } + + return null; + } internal static async ValueTask?> AuthenticateAsync(ICloudStateTransaction transaction, CloudDirectoryMoveProof proof, CancellationToken cancellationToken) @@ -135,7 +221,15 @@ await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(ScopeName, [SupportedOSPlatform("windows10.0.16299")] internal static DirectoryNativeObservation Capture(nint handle, string syncRootPath) { - NativeFileMetadata metadata = WindowsFileMetadata.Read(handle); + NativeFileMetadata metadata; + try + { + metadata = WindowsFileMetadata.Read(handle); + } + catch (System.ComponentModel.Win32Exception exception) when (exception.NativeErrorCode is 1 or 50 or 87) + { + throw new CloudDirectoryEvidenceUnavailableException("Storage cannot supply complete directory metadata.", exception); + } if (!metadata.Directory || metadata.DeletePending || metadata.PlaceholderState == CfPlaceholderState.Invalid || !metadata.PlaceholderState.HasFlag(CfPlaceholderState.Placeholder)) { @@ -143,8 +237,17 @@ internal static DirectoryNativeObservation Capture(nint handle, string syncRootP } _ = CloudPlaceholderIdentity.Decode(metadata.PlaceholderIdentity); - return new DirectoryNativeObservation(CloudLocalFileBindingPlatform.Read(handle, syncRootPath), - metadata.PlaceholderIdentity, WindowsFileMetadata.ReadFinalPath(handle)); + CloudLocalFileBinding binding; + try + { + binding = CloudLocalFileBindingPlatform.Read(handle, syncRootPath); + } + catch (NotSupportedException exception) + { + throw new CloudDirectoryEvidenceUnavailableException("Storage cannot supply a complete directory binding.", exception); + } + + return new DirectoryNativeObservation(binding, metadata.PlaceholderIdentity, WindowsFileMetadata.ReadFinalPath(handle)); } internal static async ValueTask PrepareAsync(ICloudStateTransaction transaction, CloudDirectoryMoveProof proof, @@ -192,7 +295,7 @@ internal static byte[] EncodeMembers(Guid evidenceId, Guid rootItemId, string so CloudDirectoryEvidenceCodec.WriteText(writer, suffix.Replace('/', '\\')); if (stream.Length > 64 * 1024 * 1024) { - throw new NotSupportedException("The prepared subtree exceeds the recovery metadata limit."); + throw new CloudDirectoryEvidenceUnavailableException("The prepared subtree exceeds the recovery metadata limit."); } } @@ -207,6 +310,9 @@ internal static byte[] EncodeMembers(Guid evidenceId, Guid rootItemId, string so internal sealed record CloudDirectoryMember(Guid ItemId, CloudItemKind Kind, string Suffix); +internal sealed class CloudDirectoryEvidenceUnavailableException(string message, Exception? innerException = null) + : NotSupportedException(message, innerException); + internal sealed record DirectoryNativeObservation(CloudLocalFileBinding Binding, byte[] Identity, string ActualPath) { internal bool Matches(DirectoryNativeObservation other) => Binding == other.Binding && diff --git a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs index d109353..2519270 100644 --- a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs +++ b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs @@ -85,7 +85,7 @@ internal static async ValueTask RetainProjectionAsync(ICloudStateTransaction tra // recovery evidence on storage that cannot report them. return; } - catch (NotSupportedException) + catch (CloudDirectoryEvidenceUnavailableException) { return; } diff --git a/src/CfSharp/Placeholders/CloudPlaceholderOptions.cs b/src/CfSharp/Placeholders/CloudPlaceholderOptions.cs index 32827b5..3825d0a 100644 --- a/src/CfSharp/Placeholders/CloudPlaceholderOptions.cs +++ b/src/CfSharp/Placeholders/CloudPlaceholderOptions.cs @@ -269,8 +269,25 @@ public CloudMoveOptions(bool replaceExisting) ReplaceExisting = replaceExisting; } + /// Initializes a directory move or recovery with its original durable pre-move proof. + /// Owned evidence for this exact source and intended destination. + /// The proof is null. + /// + /// Does not permit replacement. The immutable value owns no native protection; the move + /// authenticates its original store record and verifies the actual object. Use only for + /// directories; files retain the boolean constructor and existing replacement behavior. + /// + public CloudMoveOptions(CloudDirectoryMoveProof directoryMoveProof) + { + ArgumentNullException.ThrowIfNull(directoryMoveProof); + DirectoryMoveProof = directoryMoveProof; + } + /// Gets whether an existing destination file may be replaced. public bool ReplaceExisting { get; } + + /// Gets optional durable directory move evidence, or null for normal automatic preparation. + public CloudDirectoryMoveProof? DirectoryMoveProof { get; } } /// Controls explicit recursive execution over materialized local entries. diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveReplayTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveReplayTests.cs new file mode 100644 index 0000000..b5687c9 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveReplayTests.cs @@ -0,0 +1,183 @@ +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RetainedPreRenameBindingRecoversWithoutCallerPreparation(bool rootAlreadyProjected) + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Moved")); + if (rootAlreadyProjected) + { + await using ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync(); + CloudItemState original = (await transaction.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!; + await transaction.Items.UpsertAsync(new(original.ItemId, original.RemoteId, "Moved", original.Kind, + "current", original.LocalFileId, false, DateTimeOffset.UtcNow)); + await transaction.CommitAsync(); + } + + CloudItemMoveResult recovered = await source.MoveToAsync(root.FileSystem.Root, "Moved"); + Assert.Equal(rootAlreadyProjected ? 1 : 2, recovered.DurableStateEntriesUpdated); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, recovered.DirectoryReconciliation!.Outcome); + Assert.True(recovered.DirectoryReconciliation.NativeMoveObserved); + Assert.False(Directory.Exists(source.FullPath)); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.NotNull(await verify.Items.GetByRelativePathAsync("Moved\\child.txt")); + } + + [Fact] + public async Task RemoteDirectoryMoveKeepsItsExistingRevisionAndSubtreeBehavior() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + await root.FileSystem.GetDirectory("Docs").UpdatePlaceholderAsync( + CloudPlaceholderPatch.CreateBuilder().WithInSyncState(true).Build()); + CloudRemoteChange change = new("move-directory", CloudRemoteChangeKind.Move, "directory", "r2", + CloudItemKind.Directory, "Moved", itemId: root.DirectoryIdentity.ItemId, previousRelativePath: "Docs"); + CloudRemoteApplyResult result = await root.FileSystem.ApplyRemoteChangesAsync( + new CloudRemoteChangeBatch("directory-move-batch", ReadOnlyMemory.Empty, [change], new byte[] { 1 })); + Assert.Equal(CloudRemoteApplyEntryStatus.Applied, Assert.Single(result.Entries).Status); + Assert.Equal(new byte[] { 1 }, result.SafeCursor.ToArray()); + Assert.False(Directory.Exists(Path.Combine(root.RootPath, "Docs"))); + Assert.True(Directory.Exists(Path.Combine(root.RootPath, "Moved"))); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + CloudItemState directory = (await verify.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!; + Assert.Equal("Moved", directory.RelativePath); + Assert.Equal("r2", directory.RemoteRevision); + Assert.NotNull(await verify.Items.GetByRelativePathAsync("Moved\\child.txt")); + } + + [Fact] + public async Task ManagedMoveAutomaticallyPreparesAndRetriesWithoutAnotherNativeMove() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + int commits = 0; + IOException error = new("Projection commit fault."); + root.Faults.BeforeCommit = () => + { + if (++commits == 2) + { + throw error; + } + }; + CloudItemCoordinationException failed = await Assert.ThrowsAsync(() => + source.MoveToAsync(root.FileSystem.Root, "Moved").AsTask()); + Assert.Same(error, failed.InnerException); + Assert.True(failed.DirectoryReconciliation!.NativeMoveObserved); + Assert.False(failed.DirectoryReconciliation.DurableProjectionCommitted); + Assert.Equal(Path.Combine(root.RootPath, "Moved"), failed.DestinationPath); + Assert.False(Directory.Exists(source.FullPath)); + root.Faults.BeforeCommit = null; + CloudItemMoveResult recovered = await source.MoveToAsync(root.FileSystem.Root, "Moved"); + Assert.Equal(2, recovered.DurableStateEntriesUpdated); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, recovered.DirectoryReconciliation!.Outcome); + CloudItemMoveResult duplicate = await source.MoveToAsync(root.FileSystem.Root, "Moved"); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.AlreadyProjected, duplicate.DirectoryReconciliation!.Outcome); + Assert.Equal(0, duplicate.DurableStateEntriesUpdated); + Assert.Equal(root.DirectoryIdentity.ItemId, recovered.Snapshot.ItemId); + } + + [Fact] + public async Task ExplicitProofRecoversExternallyMovedPartialRootAndRejectsAnotherDestination() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + await Assert.ThrowsAsync(() => source.MoveToAsync(root.FileSystem.Root, "Other", new CloudMoveOptions(proof)).AsTask()); + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Moved")); + await using (ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync()) + { + CloudItemState original = (await transaction.Items.GetByItemIdAsync(proof.RootItemId))!; + await transaction.Items.UpsertAsync(new(original.ItemId, original.RemoteId, "Moved", original.Kind, + "current", original.LocalFileId, false, DateTimeOffset.UtcNow)); + await transaction.CommitAsync(); + } + + CloudItemMoveResult result = await source.MoveToAsync(root.FileSystem.Root, "Moved", new CloudMoveOptions(proof)); + Assert.Equal(1, result.DurableStateEntriesUpdated); + Assert.Equal("current", result.Snapshot.RemoteRevision); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, result.DirectoryReconciliation!.Outcome); + } + + [Fact] + public async Task MissingProofAndReplacedTargetCannotUseLegacyDirectoryGuessing() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + // Simulate a legacy directory with no pre-rename native evidence. Durable paths alone + // are insufficient, even when the destination has a valid placeholder identity. + await using (ICloudStateTransaction legacy = await root.Store.BeginTransactionAsync()) + { + await legacy.Checkpoints.RemoveAsync(CloudDirectoryProvenance.BindingName(root.DirectoryIdentity.ItemId)); + await legacy.Checkpoints.RemoveAsync(CloudDirectoryProvenance.MembersName(root.DirectoryIdentity.ItemId)); + await legacy.CommitAsync(); + } + + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Moved")); + await Assert.ThrowsAsync(() => source.MoveToAsync(root.FileSystem.Root, "Moved").AsTask()); + Directory.Move(Path.Combine(root.RootPath, "Moved"), source.FullPath); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Original")); + await root.FileSystem.Root.CreatePlaceholderAsync(CloudDirectoryPlaceholderSpec.CreateBuilder("Moved", "unrelated") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build()); + await Assert.ThrowsAsync(() => source.MoveToAsync(root.FileSystem.Root, "Moved", new CloudMoveOptions(proof)).AsTask()); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal("Docs", (await verify.Items.GetByItemIdAsync(proof.RootItemId))!.RelativePath); + Assert.Equal("unrelated", (await verify.Items.GetByRelativePathAsync("Moved"))!.RemoteId); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + } + + [Fact] + public async Task UnsupportedDurableProtocolNeverFallsBackToUnverifiedNativeMovement() + { + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + return; + } + + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + await using (ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync()) + { + byte[] future = new byte[20]; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(future, 2); + await transaction.Checkpoints.UpsertAsync(new(CloudDirectoryMoveEvidence.ScopeName, future, DateTimeOffset.UtcNow)); + await transaction.CommitAsync(); + } + + await Assert.ThrowsAsync(() => root.FileSystem.GetDirectory("Docs") + .MoveToAsync(root.FileSystem.Root, "Moved").AsTask()); + Assert.True(Directory.Exists(Path.Combine(root.RootPath, "Docs"))); + Assert.False(Directory.Exists(Path.Combine(root.RootPath, "Moved"))); + } +} From 56454c4baaf447f648c053c28a37dd07ddbc99bc Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:18:59 +0800 Subject: [PATCH 13/40] fix(local-changes): project verified directory rename subtrees atomically --- docs/local-change-feed.md | 18 ++ ...dFileSystem.DirectoryMoveReconciliation.cs | 64 +---- src/CfSharp/FileSystem/CloudFileSystem.cs | 3 +- .../CloudLocalChangeFeed.DirectoryMoves.cs | 114 +++++++++ .../CloudLocalChangeFeed.Scans.cs | 2 + .../LocalChanges/CloudLocalChangeFeed.cs | 138 +++++++++-- .../CloudDirectoryNativeValidation.cs | 78 ++++++ .../CloudDirectoryMoveFeedTests.cs | 223 ++++++++++++++++++ .../CloudLocalChangeNamespaceTests.cs | 53 +++++ 9 files changed, 617 insertions(+), 76 deletions(-) create mode 100644 src/CfSharp/LocalChanges/CloudLocalChangeFeed.DirectoryMoves.cs create mode 100644 src/CfSharp/Persistence/CloudDirectoryNativeValidation.cs create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveFeedTests.cs create mode 100644 tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeNamespaceTests.cs diff --git a/docs/local-change-feed.md b/docs/local-change-feed.md index 8c33ff4..c56d04c 100644 --- a/docs/local-change-feed.md +++ b/docs/local-change-feed.md @@ -89,6 +89,24 @@ or remains observable as a new rescan condition; finish reconciliation before re ## Provider echoes +Directory renames use retained library-captured native bindings and historical membership to +validate the target. Paths of the complete known subtree, its immutable move receipt, the new +formal Move operation, and the feed checkpoint commit together. Current revisions, tombstones, +and pending operations keep their original values. An old operation's paths and observation time +describe its historical event; projecting today's paths does not rewrite or acknowledge it. + +Missing evidence, an unrelated target, unknown membership, or a delayed rename whose object has +already moved again retains an unresolved formal Move with a nullable item association and requires +full reconciliation. The feed never deletes an unrelated destination row to make a rename fit. +Explicit full-rescan acknowledgement clears the reconciliation fence while leaving those original +operations pending. Dispatch and acknowledge them only after their meaning has been accepted. +Native changes racing the transaction can require reconciliation even after its projection committed; +the committed operation retains its original ID and sequence. Native notifications remain advisory. + +For an existing partial projection, use the original source directory's `MoveToAsync` or the +prepared-proof recovery described in [placeholders](placeholders.md). A matching path or remote ID +alone cannot establish historical native ownership. + Provider-originated writes can be wrapped by `SuppressProviderEchoAsync` so they do not become uploads. Hydration, pinning, and availability transitions are not local upload operations by themselves. diff --git a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs index 939676d..52e206b 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs @@ -174,69 +174,13 @@ private static T ObserveMoveNative(Func observe) } private (List<(string Path, NativeFileMetadata Metadata)> Observations, bool RequiresRescan) ObserveDirectoryMembers( - CloudDirectoryMoveProof proof, IReadOnlyList members, IReadOnlyList current) - { - Dictionary known = members.ToDictionary(member => member.ItemId); - List<(string Path, NativeFileMetadata Metadata)> observations = []; - bool requiresRescan = false; - foreach (CloudItemState item in current.Where(item => item.ItemId != proof.RootItemId)) - { - string path = Path.Combine(SyncRootPath, CloudDirectoryStateProjection.MemberPath(proof.DestinationRelativePath, known[item.ItemId].Suffix)); - using CloudPathHandleLease parents = CloudPathHandleLease.OpenParentChains(SyncRootPath, [path]); - NativeFileMetadata metadata; - try - { - using SafeFileHandle handle = WindowsFileMetadata.Open(path); - metadata = WindowsFileMetadata.Read(handle.DangerousGetHandle()); - } - catch (Win32Exception exception) when (item.IsTombstone && exception.NativeErrorCode is 2 or 3) - { - continue; - } - - if (item.IsTombstone) - { - requiresRescan = true; - continue; - } - - if (metadata.DeletePending || metadata.Directory != (item.Kind == CloudItemKind.Directory) || - (metadata.PlaceholderIdentity.Length != 0 && CloudPlaceholderIdentity.Decode(metadata.PlaceholderIdentity).ItemId != item.ItemId) || - (metadata.PlaceholderIdentity.Length == 0 && !item.RemoteId.StartsWith("local:", StringComparison.Ordinal))) - { - throw new CloudDirectoryProjectionConflictException("A native descendant no longer matches its known durable membership."); - } - - requiresRescan |= metadata.PlaceholderIdentity.Length == 0; - observations.Add((path, metadata)); - } - - return (observations, requiresRescan); - } + CloudDirectoryMoveProof proof, IReadOnlyList members, IReadOnlyList current) => + CloudDirectoryNativeValidation.ObserveMembers(SyncRootPath, proof, members, current); private void ValidateDirectoryObservation(SafeFileHandle guard, DirectoryNativeObservation expected, - List<(string Path, NativeFileMetadata Metadata)> children) - { - if (!expected.Matches(CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), SyncRootPath))) - { - throw new CloudDirectoryProjectionConflictException("The native directory changed during projection."); - } - - foreach ((string path, NativeFileMetadata original) in children) - { - using CloudPathHandleLease parents = CloudPathHandleLease.OpenParentChains(SyncRootPath, [path]); - using SafeFileHandle handle = WindowsFileMetadata.Open(path); - NativeFileMetadata actual = WindowsFileMetadata.Read(handle.DangerousGetHandle()); - if (original.Identity.VolumeSerialNumber != actual.Identity.VolumeSerialNumber || original.Identity.FileId != actual.Identity.FileId || - original.Directory != actual.Directory || actual.DeletePending || - !original.PlaceholderIdentity.AsSpan().SequenceEqual(actual.PlaceholderIdentity)) - { - throw new CloudDirectoryProjectionConflictException("A native descendant changed during projection; reconcile the retained journal."); - } - } - } + List<(string Path, NativeFileMetadata Metadata)> children) => + CloudDirectoryNativeValidation.Validate(SyncRootPath, guard, expected, children); } - internal sealed class CloudDirectoryNativeObservationException : IOException { internal CloudDirectoryNativeObservationException(Exception inner, int hresult, int? win32ErrorCode) diff --git a/src/CfSharp/FileSystem/CloudFileSystem.cs b/src/CfSharp/FileSystem/CloudFileSystem.cs index da16425..dffc2ae 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.cs @@ -115,7 +115,8 @@ public CloudLocalChangeFeed CreateLocalChangeFeed( _localChangeFeed = null; } } - }); + }, + async (scopes, cancellationToken) => await AcquireOperationAsync(scopes, cancellationToken).ConfigureAwait(false)); } } diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.DirectoryMoves.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.DirectoryMoves.cs new file mode 100644 index 0000000..c496711 --- /dev/null +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.DirectoryMoves.cs @@ -0,0 +1,114 @@ +using System.ComponentModel; +using System.Runtime.Versioning; + +using CfSharp.Native; + +using Microsoft.Win32.SafeHandles; + +namespace CfSharp; + +public sealed partial class CloudLocalChangeFeed +{ + internal const string NamespaceObservationsPrefix = "cfsharp/namespace/observations"; + + [SupportedOSPlatform("windows10.0.16299")] + private async ValueTask ProjectDirectoryObservationAsync(ICloudStateTransaction transaction, + CloudDirectoryMoveProof proof, CloudItemPath path, CloudItemPath previousPath, DateTimeOffset observedAt, + CancellationToken cancellationToken) + { + IReadOnlyList? members = await CloudDirectoryMoveEvidence.AuthenticateAsync(transaction, + proof, cancellationToken).ConfigureAwait(false); + if (members is null) + { + throw new CloudNamespaceObservationUncertainException(true); + } + + CloudPathHandleLease? parents = null; + SafeFileHandle? guard = null; + try + { + parents = CloudPathHandleLease.OpenParentChains(_syncRootPath, [path.FullPath, previousPath.FullPath]); + guard = WindowsFileMetadata.Open(path.FullPath, preventDelete: true); + DirectoryNativeObservation target = CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), _syncRootPath); + if (!CloudDirectoryNativeValidation.MatchesProof(target, proof) || + !string.Equals(Path.GetRelativePath(_syncRootPath, target.ActualPath), proof.DestinationRelativePath, StringComparison.Ordinal)) + { + throw new CloudNamespaceObservationUncertainException(true); + } + + try + { + using SafeFileHandle sourceHandle = WindowsFileMetadata.Open(previousPath.FullPath); + DirectoryNativeObservation source = CloudDirectoryMoveEvidence.Capture(sourceHandle.DangerousGetHandle(), _syncRootPath); + if (!CloudDirectoryNativeValidation.MatchesProof(source, proof) || + string.Equals(Path.GetRelativePath(_syncRootPath, source.ActualPath), proof.SourceRelativePath, StringComparison.Ordinal)) + { + throw new CloudNamespaceObservationUncertainException(true); + } + } + catch (Win32Exception exception) when (exception.NativeErrorCode is 2 or 3) + { + // An absent source is expected after a rename; the retained binding proves origin. + } + + VerifiedDirectoryProjection projection = await CloudDirectoryStateProjection.ProjectVerifiedAsync(transaction, + proof, members, observedAt, cancellationToken).ConfigureAwait(false); + var children = CloudDirectoryNativeValidation.ObserveMembers(_syncRootPath, proof, members, projection.CurrentItems); + if (!projection.AlreadyCompleted) + { + await CloudDirectoryProvenance.ProjectPathsAsync(transaction, proof, members, projection.CurrentItems, + cancellationToken).ConfigureAwait(false); + } + + DirectoryFeedProjection result = new(_syncRootPath, proof.RootItemId, guard, parents, target, + children.Observations, children.RequiresRescan); + guard = null; + parents = null; + return result; + } + catch (Exception exception) when (exception is Win32Exception or NativeFileException or InvalidOperationException or CloudDirectoryEvidenceUnavailableException) + { + // The caller disposes this transaction before persisting an unresolved observation. + // Never commit partial projector writes after a failed native or membership check. + throw new CloudNamespaceObservationUncertainException(true, exception); + } + finally + { + guard?.Dispose(); + parents?.Dispose(); + } + } + + [SupportedOSPlatform("windows10.0.16299")] + private sealed class DirectoryFeedProjection(string syncRootPath, Guid itemId, SafeFileHandle guard, + CloudPathHandleLease parents, DirectoryNativeObservation expected, + List<(string Path, NativeFileMetadata Metadata)> children, bool requiresRescan) : IDisposable + { + internal Guid ItemId => itemId; + internal bool RequiresRescan => requiresRescan; + + internal void Validate() + { + try + { + CloudDirectoryNativeValidation.Validate(syncRootPath, guard, expected, children); + } + catch (Exception exception) when (exception is Win32Exception or NativeFileException or InvalidOperationException or CloudDirectoryEvidenceUnavailableException) + { + throw new CloudNamespaceObservationUncertainException(true, exception); + } + } + + public void Dispose() + { + guard.Dispose(); + parents.Dispose(); + } + } +} + +internal sealed class CloudNamespaceObservationUncertainException(bool isDirectory, Exception? inner = null) + : IOException("The namespace observation requires full reconciliation.", inner) +{ + internal bool IsDirectory => isDirectory; +} diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs index 1c244d8..27099ee 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs @@ -145,6 +145,8 @@ private static async ValueTask RequiresCreationReconciliationAsync(ICloudS CancellationToken cancellationToken) => (await transaction.Checkpoints.ListAsync(RemoteCreationIntent.ObservationsPrefix, cancellationToken) .ConfigureAwait(false)).Count != 0 || + (await transaction.Checkpoints.ListAsync(NamespaceObservationsPrefix, cancellationToken) + .ConfigureAwait(false)).Count != 0 || (await transaction.Checkpoints.ListAsync(RemoteCreationIntent.Prefix, cancellationToken) .ConfigureAwait(false)).Any(value => !RemoteCreationIntent.Decode(value.Value).Committed); } diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs index 91dffa8..e10a83a 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs @@ -28,6 +28,7 @@ public sealed partial class CloudLocalChangeFeed : IDisposable, IAsyncDisposable private readonly CloudLocalChangeFeedOptions _options; private readonly ILocalChangeSource _source; private readonly Action? _onDisposed; + private readonly Func, CancellationToken, ValueTask>? _acquireOperation; private readonly Channel _sourceEvents; private readonly Channel _availability = Channel.CreateUnbounded( new UnboundedChannelOptions @@ -55,7 +56,8 @@ internal CloudLocalChangeFeed( ICloudStateStore stateStore, CloudLocalChangeFeedOptions options, ILocalChangeSource source, - Action? onDisposed = null) + Action? onDisposed = null, + Func, CancellationToken, ValueTask>? acquireOperation = null) { _syncRootPath = Path.TrimEndingDirectorySeparator(Path.GetFullPath(syncRootPath)); _stateStore = stateStore; @@ -63,6 +65,7 @@ internal CloudLocalChangeFeed( _options.Validate(); _source = source; _onDisposed = onDisposed; + _acquireOperation = acquireOperation; _sourceEvents = Channel.CreateBounded( new BoundedChannelOptions(options.BufferCapacity) { @@ -261,6 +264,13 @@ public async ValueTask AcknowledgeFullRescanAsync(CancellationToken cancellation await transaction.Checkpoints.RemoveAsync(observation.Name, cancellationToken).ConfigureAwait(false); } + foreach (CloudStateCheckpoint observation in await transaction.Checkpoints + .ListAsync(NamespaceObservationsPrefix, cancellationToken).ConfigureAwait(false)) + { + // Reconciliation clears the fence, not the original pending journal operation. + await transaction.Checkpoints.RemoveAsync(observation.Name, cancellationToken).ConfigureAwait(false); + } + LocalChangeCheckpoint checkpoint = await ReadCheckpointAsync( transaction, cancellationToken).ConfigureAwait(false); @@ -490,6 +500,15 @@ private async Task ProcessEventsAsync() } catch (Exception exception) { + SignalLoss(); + try + { + await MarkRescanRequiredAsync(CancellationToken.None).ConfigureAwait(false); + } + catch + { + // Preserve the worker's original failure when the store also cannot record loss. + } Volatile.Write(ref _failure, exception); _availability.Writer.TryWrite(true); } @@ -628,6 +647,47 @@ private async ValueTask PersistChangeAsync( } DateTimeOffset observedAt = DateTimeOffset.UtcNow; + Guid operationId = Guid.NewGuid(); + // Acquire facade scopes before the store gate, preserving the facade's lock order. + // Standalone deterministic feeds still use the native guards inside the projector. + using IDisposable? operation = kind == CloudLocalChangeKind.Move && previousPath is not null && _acquireOperation is not null + ? await _acquireOperation([CloudItemOperationScope.Subtree(previousPath.Value.FullPath), + CloudItemOperationScope.Subtree(path.FullPath)], cancellationToken).ConfigureAwait(false) + : null; + try + { + await PersistChangeCoreAsync(kind, path, previousPath, operationId, observedAt, null, cancellationToken).ConfigureAwait(false); + } + catch (CloudNamespaceObservationUncertainException exception) + { + // The original transaction has rolled back (or committed before a late native + // check failed). Reuse its acknowledgement identity when retaining uncertainty. + SignalLoss(); + await PersistChangeCoreAsync(kind, path, previousPath, operationId, observedAt, exception.IsDirectory, + cancellationToken).ConfigureAwait(false); + } + catch (IOException) when (kind == CloudLocalChangeKind.Move) + { + SignalLoss(); + try + { + await PersistChangeCoreAsync(kind, path, previousPath, operationId, observedAt, + Directory.Exists(path.FullPath), cancellationToken).ConfigureAwait(false); + } + catch + { + // A persistent store failure may prevent durable capture. The original error + // still terminates the worker and requires recovery rather than reporting success. + } + + throw; + } + } + + private async ValueTask PersistChangeCoreAsync(CloudLocalChangeKind kind, CloudItemPath path, + CloudItemPath? previousPath, Guid operationId, DateTimeOffset observedAt, bool? uncertainDirectory, + CancellationToken cancellationToken) + { await using ICloudStateTransaction transaction = await _stateStore .BeginTransactionAsync(cancellationToken).ConfigureAwait(false); LocalChangeCheckpoint checkpoint = await ReadCheckpointAsync(transaction, cancellationToken) @@ -676,13 +736,28 @@ await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint( CloudItemState? observedState = kind == CloudLocalChangeKind.Delete ? current : previous ?? current; + bool directory = observedState?.Kind == CloudItemKind.Directory || Directory.Exists(path.FullPath); + using DirectoryFeedProjection? directoryProjection = uncertainDirectory is null && kind == CloudLocalChangeKind.Move && previousPath is not null + ? await TryProjectDirectoryAsync().ConfigureAwait(false) : null; + if (uncertainDirectory is null && kind == CloudLocalChangeKind.Move && previous is not null && current is not null && + previous.ItemId != current.ItemId) + { + throw new CloudNamespaceObservationUncertainException(directory); + } + + if (uncertainDirectory is null && kind == CloudLocalChangeKind.Move && directoryProjection is null && + observedState is null && !File.Exists(path.FullPath)) + { + throw new CloudNamespaceObservationUncertainException(directory); + } + IReadOnlyList suppressions = await transaction.EchoSuppressions .ListActiveAsync(observedAt, cancellationToken).ConfigureAwait(false); CloudEchoSuppressionState? suppression = suppressions.FirstOrDefault(candidate => RemoteCreationIntent.IsCreationEcho(candidate) ? MatchesRemoteCreationEcho(candidate, operationKind, path, observedState) : candidate.Matches(operationKind, path.RelativePath, previousPath?.RelativePath, observedState?.ItemId)); - if (suppression is not null) + if (suppression is not null && uncertainDirectory is null && directoryProjection?.RequiresRescan != true) { if (suppression.RemainingObservations == 1) { @@ -704,7 +779,9 @@ await UpsertCheckpointAsync( checkpoint.RequiresFullRescan, observedAt, cancellationToken).ConfigureAwait(false); + directoryProjection?.Validate(); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + directoryProjection?.Validate(); SignalAvailable(); return; } @@ -715,14 +792,9 @@ await UpsertCheckpointAsync( CloudItemState? state = kind == CloudLocalChangeKind.Delete ? current : previous ?? current; - Guid? itemId = state?.ItemId; - if (kind == CloudLocalChangeKind.Move && previous is not null && current is not null && - previous.ItemId != current.ItemId) - { - await transaction.Items.RemoveAsync(current.ItemId, cancellationToken).ConfigureAwait(false); - } + Guid? itemId = uncertainDirectory is not null ? null : directoryProjection?.ItemId ?? state?.ItemId; - if (state is null && kind != CloudLocalChangeKind.Delete) + if (uncertainDirectory is null && directoryProjection is null && state is null && kind != CloudLocalChangeKind.Delete) { itemId = Guid.NewGuid(); state = new CloudItemState( @@ -736,7 +808,7 @@ await UpsertCheckpointAsync( observedAt); } - if (state is not null) + if (uncertainDirectory is null && directoryProjection is null && state is not null) { CloudItemState updated = new( state.ItemId, @@ -753,25 +825,61 @@ await UpsertCheckpointAsync( LocalChangePayload payload = new( path.RelativePath, previousPath?.RelativePath, - state?.Kind == CloudItemKind.Directory || - (state is null && Directory.Exists(path.FullPath)), + uncertainDirectory ?? (directoryProjection is not null || directory), observedAt); - await transaction.Operations.EnqueueAsync( + if (await transaction.Operations.GetAsync(operationId, cancellationToken).ConfigureAwait(false) is null) + { + await transaction.Operations.EnqueueAsync( new CloudOperationJournalEntry( - Guid.NewGuid(), + operationId, operationKind, itemId, payload.Encode(), observedAt), cancellationToken).ConfigureAwait(false); + } + + bool requiresRescan = uncertainDirectory is not null || directoryProjection?.RequiresRescan == true; + long namespaceGeneration = requiresRescan ? SignalLoss() : Volatile.Read(ref _lossGeneration); + if (requiresRescan) + { + await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(NamespaceObservationsPrefix + "/" + operationId.ToString("N"), + payload.Encode(), observedAt), cancellationToken).ConfigureAwait(false); + } + await UpsertCheckpointAsync( transaction, observation, - checkpoint.RequiresFullRescan, + checkpoint.RequiresFullRescan || requiresRescan, observedAt, cancellationToken).ConfigureAwait(false); + directoryProjection?.Validate(); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + if (requiresRescan) + { + Volatile.Write(ref _persistedLossGeneration, namespaceGeneration); + } + + directoryProjection?.Validate(); SignalAvailable(); + + async ValueTask TryProjectDirectoryAsync() + { + CloudDirectoryMoveProof? proof = await CloudDirectoryMoveEvidence.GetOrPrepareRetainedIntentAsync(transaction, + previousPath!.Value.RelativePath, path.RelativePath, cancellationToken).ConfigureAwait(false); + if (proof is null && !directory) + { + return null; + } + + if (proof is null || !OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + throw new CloudNamespaceObservationUncertainException(true); + } + + return await ProjectDirectoryObservationAsync(transaction, proof, path, previousPath.Value, observedAt, + cancellationToken).ConfigureAwait(false); + } } private static bool MatchesRemoteCreationEcho(CloudEchoSuppressionState suppression, diff --git a/src/CfSharp/Persistence/CloudDirectoryNativeValidation.cs b/src/CfSharp/Persistence/CloudDirectoryNativeValidation.cs new file mode 100644 index 0000000..e9ec57d --- /dev/null +++ b/src/CfSharp/Persistence/CloudDirectoryNativeValidation.cs @@ -0,0 +1,78 @@ +using System.ComponentModel; +using System.Runtime.Versioning; + +using CfSharp.Native; + +using Microsoft.Win32.SafeHandles; + +namespace CfSharp; + +[SupportedOSPlatform("windows10.0.16299")] +internal static class CloudDirectoryNativeValidation +{ + internal static bool MatchesProof(DirectoryNativeObservation observation, CloudDirectoryMoveProof proof) => + observation.Binding == proof.ExpectedBinding && observation.Identity.AsSpan().SequenceEqual(proof.ExpectedPlaceholderIdentity.Span); + + internal static (List<(string Path, NativeFileMetadata Metadata)> Observations, bool RequiresRescan) ObserveMembers( + string syncRootPath, CloudDirectoryMoveProof proof, IReadOnlyList members, IReadOnlyList current) + { + Dictionary known = members.ToDictionary(member => member.ItemId); + List<(string Path, NativeFileMetadata Metadata)> observations = []; + bool requiresRescan = false; + foreach (CloudItemState item in current.Where(item => item.ItemId != proof.RootItemId)) + { + string path = Path.Combine(syncRootPath, CloudDirectoryStateProjection.MemberPath(proof.DestinationRelativePath, known[item.ItemId].Suffix)); + using CloudPathHandleLease parents = CloudPathHandleLease.OpenParentChains(syncRootPath, [path]); + NativeFileMetadata metadata; + try + { + using SafeFileHandle handle = WindowsFileMetadata.Open(path); + metadata = WindowsFileMetadata.Read(handle.DangerousGetHandle()); + } + catch (Win32Exception exception) when (item.IsTombstone && exception.NativeErrorCode is 2 or 3) + { + continue; + } + + if (item.IsTombstone) + { + requiresRescan = true; + continue; + } + + if (metadata.DeletePending || metadata.Directory != (item.Kind == CloudItemKind.Directory) || + (metadata.PlaceholderIdentity.Length != 0 && CloudPlaceholderIdentity.Decode(metadata.PlaceholderIdentity).ItemId != item.ItemId) || + (metadata.PlaceholderIdentity.Length == 0 && !item.RemoteId.StartsWith("local:", StringComparison.Ordinal))) + { + throw new CloudDirectoryProjectionConflictException("A native descendant no longer matches its known durable membership."); + } + + requiresRescan |= metadata.PlaceholderIdentity.Length == 0; + observations.Add((path, metadata)); + } + + return (observations, requiresRescan); + } + + internal static void Validate(string syncRootPath, SafeFileHandle guard, DirectoryNativeObservation expected, + List<(string Path, NativeFileMetadata Metadata)> children) + { + if (!expected.Matches(CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), syncRootPath))) + { + throw new CloudDirectoryProjectionConflictException("The native directory changed during projection."); + } + + foreach ((string path, NativeFileMetadata original) in children) + { + using CloudPathHandleLease parents = CloudPathHandleLease.OpenParentChains(syncRootPath, [path]); + using SafeFileHandle handle = WindowsFileMetadata.Open(path); + NativeFileMetadata actual = WindowsFileMetadata.Read(handle.DangerousGetHandle()); + if (original.Identity.VolumeSerialNumber != actual.Identity.VolumeSerialNumber || original.Identity.FileId != actual.Identity.FileId || + original.Directory != actual.Directory || actual.DeletePending || + !original.PlaceholderIdentity.AsSpan().SequenceEqual(actual.PlaceholderIdentity)) + { + throw new CloudDirectoryProjectionConflictException("A native descendant changed during projection; reconcile the retained journal."); + } + } + } +} diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveFeedTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveFeedTests.cs new file mode 100644 index 0000000..b7622e8 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveFeedTests.cs @@ -0,0 +1,223 @@ +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Fact] + public async Task NativeWatcherProjectsWholeKnownSubtreeAndRetainsPendingJournal() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudOperationJournalEntry original = await SeedPendingChildAsync(root); + CloudLocalChangeFeed feed = root.FileSystem.CreateLocalChangeFeed(); + await feed.StartAsync(); + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(15)); + CloudLocalChange? move = null; + while (move is null) + { + CloudLocalChangeScan scan = await feed.BeginScanAsync(timeout.Token); + Assert.False(scan.RequiresFullRescan); + CloudLocalChangePage page = await feed.ReadPageAsync(scan, 0, 4096, timeout.Token); + move = page.Changes.FirstOrDefault(change => change.Kind == CloudLocalChangeKind.Move && change.RelativePath == "Moved"); + if (move is null) + { + await Task.Delay(20, timeout.Token); + } + } + + Assert.True(move.IsDirectory); + Assert.Equal(root.DirectoryIdentity.ItemId, move.ItemId); + await AssertProjectedChildAsync(root, "Moved", original); + } + + [Fact] + public async Task DelayedHistoricalRenamePreservesLatestProjectionAndOriginalOperations() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudOperationJournalEntry original = await SeedPendingChildAsync(root); + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + await ProcessRenameAsync(root, "Docs", "Moved"); + Directory.Move(Path.Combine(root.RootPath, "Moved"), Path.Combine(root.RootPath, "Latest")); + await ProcessRenameAsync(root, "Moved", "Latest"); + await ProcessRenameAsync(root, "Docs", "Moved"); + await AssertProjectedChildAsync(root, "Latest", original); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + CloudOperationJournalEntry[] moves = (await verify.Operations.ListAsync(100)).Where(row => row.Kind == CloudStateOperationKind.Move).ToArray(); + Assert.Equal(3, moves.Length); + Assert.Null(moves[^1].ItemId); + Assert.Single(await verify.Checkpoints.ListAsync(CloudLocalChangeFeed.NamespaceObservationsPrefix)); + Assert.True(LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + } + + [Fact] + public async Task NativeChangeAfterCommitRetainsOneOriginalMoveAndFencesPages() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudOperationJournalEntry original = await SeedPendingChildAsync(root); + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + CloudOperationJournalEntry? committedMove = null; + root.Faults.AfterCommit = () => + { + root.Faults.AfterCommit = null; + File.Delete(Path.Combine(root.RootPath, "Moved", "child.txt")); + }; + await ProcessRenameAsync(root, "Docs", "Moved"); + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + committedMove = Assert.Single((await verify.Operations.ListAsync(100)).Where(row => row.Kind == CloudStateOperationKind.Move)); + Assert.Equal(root.DirectoryIdentity.ItemId, committedMove.ItemId); + Assert.Single(await verify.Checkpoints.ListAsync(CloudLocalChangeFeed.NamespaceObservationsPrefix)); + } + + FeedSource source = new(); + await using CloudLocalChangeFeed restarted = CloudLocalChangeFeed.CreateForTesting(root.RootPath, root.Store, new(), source); + await restarted.StartAsync(); + CloudLocalChangeScan scan = await restarted.BeginScanAsync(); + Assert.True(scan.RequiresFullRescan); + Assert.Empty((await restarted.ReadPageAsync(scan, 0, 4)).Changes); + await restarted.AcknowledgeFullRescanAsync(); + Assert.True((await restarted.ReadPageAsync(scan, 0, 4)).RequiresFullRescan); + CloudLocalChangePage page = await restarted.ReadPageAsync(await restarted.BeginScanAsync(), 0, 4); + Assert.Contains(page.Changes, change => change.OperationId == committedMove.OperationId && change.Sequence == committedMove.Sequence); + await AssertProjectedChildAsync(root, "Moved", original); + } + + [Fact] + public async Task UnknownDescendantRollsBackAllProjectionAndRetainsFormalMove() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudOperationJournalEntry original = await SeedPendingChildAsync(root); + CloudDirectoryMoveProof proof = await root.FileSystem.GetDirectory("Docs").PrepareMoveAsync(root.FileSystem.Root, "Moved"); + Guid unknown = Guid.NewGuid(); + await using (ICloudStateTransaction change = await root.Store.BeginTransactionAsync()) + { + await change.Items.UpsertAsync(new(unknown, "unknown", "Docs\\unknown.txt", CloudItemKind.File, "r-new", null, false, DateTimeOffset.UtcNow)); + await change.CommitAsync(); + } + + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + await ProcessRenameAsync(root, "Docs", "Moved"); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal("Docs", (await verify.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!.RelativePath); + Assert.Equal("Docs\\unknown.txt", (await verify.Items.GetByItemIdAsync(unknown))!.RelativePath); + Assert.Equal("Docs\\child.txt", (await verify.Items.GetByItemIdAsync(original.ItemId!.Value))!.RelativePath); + Assert.Null(Assert.Single((await verify.Operations.ListAsync(100)).Where(row => row.Kind == CloudStateOperationKind.Move)).ItemId); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + Assert.NotNull(await verify.Operations.GetAsync(original.OperationId)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task FeedPreservesKnownTombstonesAndFencesPhysicallyPresentDeletedMembers(bool physicallyPresent) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + Guid childId; + await using (ICloudStateTransaction change = await root.Store.BeginTransactionAsync()) + { + CloudItemState child = (await change.Items.GetByRelativePathAsync("Docs\\child.txt"))!; + childId = child.ItemId; + await change.Items.UpsertAsync(new(childId, child.RemoteId, child.RelativePath, child.Kind, "deleted-revision", 42, true, child.UpdatedAt)); + await change.CommitAsync(); + } + + if (!physicallyPresent) + { + File.Delete(Path.Combine(root.RootPath, "Docs", "child.txt")); + } + + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + await ProcessRenameAsync(root, "Docs", "Moved"); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + CloudItemState result = (await verify.Items.GetByItemIdAsync(childId))!; + Assert.Equal("Moved\\child.txt", result.RelativePath); + Assert.True(result.IsTombstone); + Assert.Equal("deleted-revision", result.RemoteRevision); + Assert.Equal(42, result.LocalFileId); + Assert.Equal(physicallyPresent, LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + } + + [Fact] + public async Task FeedProjectionCommitFailureRollsBackSubtreeAndRetainsUnresolvedMoveForRestart() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudOperationJournalEntry original = await SeedPendingChildAsync(root); + CloudDirectoryMoveProof proof = await root.FileSystem.GetDirectory("Docs").PrepareMoveAsync(root.FileSystem.Root, "Moved"); + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + IOException failure = new("Injected directory feed commit failure."); + root.Faults.BeforeCommit = () => + { + root.Faults.BeforeCommit = null; + throw failure; + }; + await ProcessRenameAsync(root, "Docs", "Moved"); + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + Assert.Equal("Docs", (await verify.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!.RelativePath); + Assert.Equal("Docs\\child.txt", (await verify.Items.GetByItemIdAsync(original.ItemId!.Value))!.RelativePath); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + Assert.Null(Assert.Single((await verify.Operations.ListAsync(100)).Where(row => row.Kind == CloudStateOperationKind.Move)).ItemId); + } + + CloudDirectoryMoveReconciliationResult recovered = await root.FileSystem.GetDirectory("Docs").ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, recovered.Outcome); + await AssertProjectedChildAsync(root, "Moved", original); + await using CloudLocalChangeFeed restarted = CloudLocalChangeFeed.CreateForTesting(root.RootPath, root.Store, new(), new FeedSource()); + await restarted.StartAsync(); + Assert.True((await restarted.BeginScanAsync()).RequiresFullRescan); + } + + private static async Task SeedPendingChildAsync(DirectoryMoveTestRoot root) + { + await using ICloudStateTransaction transaction = await root.Store.BeginTransactionAsync(); + CloudItemState child = (await transaction.Items.GetByRelativePathAsync("Docs\\child.txt"))!; + await transaction.Items.UpsertAsync(new(child.ItemId, child.RemoteId, child.RelativePath, child.Kind, + "revision-pending", 123, child.IsTombstone, child.UpdatedAt)); + Guid id = Guid.NewGuid(); + await transaction.Operations.EnqueueAsync(new(id, CloudStateOperationKind.ContentUpdate, child.ItemId, + new LocalChangePayload(child.RelativePath, null, false, DateTimeOffset.UtcNow).Encode(), DateTimeOffset.UtcNow)); + await transaction.CommitAsync(); + await using ICloudStateTransaction read = await root.Store.BeginTransactionAsync(); + return (await read.Operations.GetAsync(id))!; + } + + private static async Task AssertProjectedChildAsync(DirectoryMoveTestRoot root, string directory, CloudOperationJournalEntry original) + { + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal(directory, (await verify.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!.RelativePath); + CloudItemState child = (await verify.Items.GetByItemIdAsync(original.ItemId!.Value))!; + Assert.Equal(directory + "\\child.txt", child.RelativePath); + Assert.Equal("revision-pending", child.RemoteRevision); + Assert.Equal(123, child.LocalFileId); + Assert.False(child.IsTombstone); + CloudOperationJournalEntry pending = (await verify.Operations.GetAsync(original.OperationId))!; + Assert.Equal(original.Sequence, pending.Sequence); + Assert.Equal(original.CreatedAt, pending.CreatedAt); + Assert.Equal(original.Payload.ToArray(), pending.Payload.ToArray()); + } + + private static async Task ProcessRenameAsync(DirectoryMoveTestRoot root, string sourcePath, string targetPath) + { + FeedSource source = new(); + await using CloudLocalChangeFeed feed = CloudLocalChangeFeed.CreateForTesting(root.RootPath, root.Store, new(), source); + await feed.StartAsync(); + await source.EmitAsync(new(LocalChangeSourceAction.RenamedOldName, sourcePath)); + await source.EmitAsync(new(LocalChangeSourceAction.RenamedNewName, targetPath)); + await feed.DisposeAsync(); + } + + private sealed class FeedSource : ILocalChangeSource + { + private Func _handler = null!; + public Task StartAsync(Func eventHandler, CancellationToken cancellationToken) + { + _handler = eventHandler; + return Task.CompletedTask; + } + + internal ValueTask EmitAsync(LocalChangeSourceEvent sourceEvent) => _handler(sourceEvent); + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeNamespaceTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeNamespaceTests.cs new file mode 100644 index 0000000..6e4edaf --- /dev/null +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeNamespaceTests.cs @@ -0,0 +1,53 @@ +namespace CfSharp.Storage.Sqlite.Tests; + +public sealed partial class CloudLocalChangeFeedTests +{ + [Theory] + [InlineData(CloudItemKind.File)] + [InlineData(CloudItemKind.Directory)] + public async Task UnprovenMoveKeepsUnrelatedTargetAndJournalUntilExplicitAcknowledgement(CloudItemKind kind) + { + await using ICloudStateStore store = await OpenStoreAsync(); + Guid sourceId = Guid.NewGuid(); + Guid targetId = Guid.NewGuid(); + await using (ICloudStateTransaction seed = await store.BeginTransactionAsync()) + { + await seed.Items.UpsertAsync(new(sourceId, "source", "before", kind, "source-revision", 10, false, DateTimeOffset.UtcNow)); + await seed.Items.UpsertAsync(new(targetId, "unrelated", "after", kind, "target-revision", 20, true, DateTimeOffset.UtcNow)); + await seed.CommitAsync(); + } + + FakeSource source = new(); + await using (CloudLocalChangeFeed feed = CreateFeed(store, source)) + { + await feed.StartAsync(); + await feed.SuppressProviderEchoAsync(CloudStateOperationKind.Move, "after", DateTimeOffset.UtcNow.AddMinutes(1), sourceId, "before", 1); + await source.EmitAsync(new(LocalChangeSourceAction.RenamedOldName, "before")); + await source.EmitAsync(new(LocalChangeSourceAction.RenamedNewName, "after")); + } + + Guid operationId; + await using (ICloudStateTransaction verify = await store.BeginTransactionAsync()) + { + Assert.Equal("before", (await verify.Items.GetByItemIdAsync(sourceId))!.RelativePath); + CloudItemState target = (await verify.Items.GetByItemIdAsync(targetId))!; + Assert.True(target.IsTombstone); + Assert.Equal("target-revision", target.RemoteRevision); + CloudOperationJournalEntry move = Assert.Single(await verify.Operations.ListAsync(10)); + Assert.Null(move.ItemId); + Assert.Equal(kind == CloudItemKind.Directory, LocalChangePayload.Decode(move.Payload).IsDirectory); + operationId = move.OperationId; + Assert.Single(await verify.EchoSuppressions.ListActiveAsync(DateTimeOffset.UtcNow)); + } + + await using CloudLocalChangeFeed restarted = CreateFeed(store, new()); + await restarted.StartAsync(); + Assert.True((await restarted.BeginScanAsync()).RequiresFullRescan); + await restarted.AcknowledgeFullRescanAsync(); + CloudLocalChangePage page = await restarted.ReadPageAsync(await restarted.BeginScanAsync(), 0, 4); + Assert.Equal(operationId, Assert.Single(page.Changes).OperationId); + await using ICloudStateTransaction retained = await store.BeginTransactionAsync(); + Assert.NotNull(await retained.Operations.GetAsync(operationId)); + Assert.Empty(await retained.Checkpoints.ListAsync(CloudLocalChangeFeed.NamespaceObservationsPrefix)); + } +} From ead18f50960386e8c16b076d318738e4fa3b5032 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:30:18 +0800 Subject: [PATCH 14/40] test(items): cover directory reconciliation object races --- docs/placeholders.md | 3 + .../CloudDirectoryMoveRaceTests.cs | 214 ++++++++++++++++++ .../CloudDirectoryMoveTests.cs | 4 +- .../NamespaceConsumer.cs | 38 ++++ .../Program.cs | 5 + 5 files changed, 262 insertions(+), 2 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveRaceTests.cs create mode 100644 tests/CfSharp.Storage.Sqlite.CrashHarness/NamespaceConsumer.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index 6b7c091..8163d57 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -36,6 +36,9 @@ native observation failures. Metadata guards stabilize the root object but do no changes or identity writes. Validation around commit detects changed known descendants; local ordinary children without native placeholder identity require a rescan before dispatch. A detected post-commit race reports the committed fact and rescan requirement rather than claiming rollback. +An approved external Cloud Files rename can change the namespace while a metadata guard remains +open. Treat the result's native observation as a checked fact at that stage, not a continuing +namespace lock; honor a later conflict/rescan even when `DurableProjectionCommitted` is true. `MoveToAsync` prepares supported managed directories before its native move. Retrying the original source/destination after a crash uses the retained indexed preparation and performs only verified diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveRaceTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveRaceTests.cs new file mode 100644 index 0000000..b0a3bff --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveRaceTests.cs @@ -0,0 +1,214 @@ +using System.Diagnostics; +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Fact] + public async Task IndependentConsumerTriggersActualRenameApprovalAndCompletion() + { + RenameProvider provider = new(); + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(provider); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + using NamespaceProcess consumer = new(source.FullPath, Path.Combine(root.RootPath, "Moved"), "rename"); + await consumer.WaitForReadyAsync(); + consumer.Go.Set(); + await consumer.WaitForExitAsync(); + Assert.Equal(0, consumer.ExitCode); + Assert.True(consumer.Attempted.WaitOne(0)); + Assert.True(consumer.Completed.WaitOne(0)); + CloudProviderRenameRequest approval = await provider.Approval.Task.WaitAsync(TimeSpan.FromSeconds(10)); + Assert.True(approval.IsDirectory && approval.SourceInScope && approval.TargetInScope); + Assert.Equal(proof.ExpectedPlaceholderIdentity.ToArray(), approval.FileIdentity.ToArray()); + CloudProviderCompletionNotification completion = await provider.Completion.Task.WaitAsync(TimeSpan.FromSeconds(10)); + Assert.EndsWith("\\Moved", completion.NormalizedPath); + Assert.EndsWith("\\Docs", completion.RelatedPath); + Assert.Equal(1, provider.Approvals); + Assert.Equal(1, provider.Completions); + CloudDirectoryMoveReconciliationResult result = await source.ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, result.Outcome); + Assert.True(result.NativeMoveObserved && result.DurableProjectionCommitted); + } + + [Theory] + [InlineData("rename")] + [InlineData("delete")] + [InlineData("create")] + [InlineData("edit")] + public async Task IndependentNamespaceCompetitionReportsNativeAndCommittedFacts(string action) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(new RenameProvider()); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + string target = Path.Combine(root.RootPath, "Moved"); + Directory.Move(source.FullPath, target); + using NamespaceProcess competitor = new(action == "rename" ? target : Path.Combine(target, "child.txt"), + action == "rename" ? Path.Combine(root.RootPath, "Elsewhere") : Path.Combine(target, "new.txt"), action); + await competitor.WaitForReadyAsync(); + root.Faults.BeforeCommit = () => + { + root.Faults.BeforeCommit = null; + competitor.Go.Set(); + competitor.WaitForExit(); + Assert.True(competitor.Attempted.WaitOne(0) && competitor.Completed.WaitOne(0)); + }; + CloudDirectoryMoveReconciliationResult result = await source.ReconcileMoveAsync(proof); + Assert.True(result.NativeMoveObserved && result.DurableProjectionCommitted); + if (action == "rename") + { + Assert.True(competitor.ExitCode is 0 or 32, $"Unexpected native consumer error: {competitor.ExitCode}"); + Assert.Equal(competitor.ExitCode == 0 ? CloudDirectoryMoveReconciliationOutcome.Conflict : CloudDirectoryMoveReconciliationOutcome.Projected, + result.Outcome); + Assert.Equal(competitor.ExitCode == 0, result.RequiresFullRescan); + Assert.Equal(competitor.ExitCode == 0, Directory.Exists(Path.Combine(root.RootPath, "Elsewhere"))); + // Cloud Files approval can permit a namespace change while our metadata handle + // remains open. The post-commit actual-path check must expose that split fact. + } + else + { + Assert.Equal(0, competitor.ExitCode); + // Truncating a placeholder can also change its native placeholder metadata. + // Those detected changes report a committed projection plus a reconciliation fence. + Assert.Equal(action is "delete" or "edit", result.RequiresFullRescan); + Assert.Equal(action is "delete" or "edit" ? CloudDirectoryMoveReconciliationOutcome.Conflict : CloudDirectoryMoveReconciliationOutcome.Projected, + result.Outcome); + } + + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal("Moved", (await verify.Items.GetByItemIdAsync(proof.RootItemId))!.RelativePath); + Assert.NotNull(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + } + + [Fact] + public async Task CopiedPlaceholderIdentityOnDifferentNativeDirectoryCannotReplaceOriginalBinding() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Away")); + Directory.Move(Path.Combine(root.RootPath, "Ordinary"), Path.Combine(root.RootPath, "Moved")); + await using (ICloudStateTransaction read = await root.Store.BeginTransactionAsync()) + { + CloudItemState original = (await read.Items.GetByItemIdAsync(proof.RootItemId))!; + await read.RollbackAsync(); + await root.FileSystem.GetDirectory("Moved").ConvertToPlaceholderAsync(root.DirectoryIdentity); + await using ICloudStateTransaction restore = await root.Store.BeginTransactionAsync(); + await restore.Items.UpsertAsync(original); + await restore.CommitAsync(); + } + + CloudItemSnapshot target = await root.FileSystem.GetDirectory("Moved").InspectAsync(); + Assert.Equal(proof.ExpectedPlaceholderIdentity.ToArray(), target.PlaceholderIdentity.ToArray()); + Assert.NotEqual(proof.ExpectedBinding, target.LocalBinding); + CloudDirectoryMoveReconciliationResult result = await source.ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Conflict, result.Outcome); + Assert.False(result.NativeMoveObserved || result.DurableProjectionCommitted); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal("Docs", (await verify.Items.GetByItemIdAsync(proof.RootItemId))!.RelativePath); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + } + + private sealed class RenameProvider : ICloudDemandProvider + { + internal int Approvals; + internal int Completions; + internal TaskCompletionSource Approval { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public ValueTask OpenReadAsync(CloudFileFetchRequest request, CancellationToken cancellationToken) => + ValueTask.FromResult(new MemoryStream()); + public ValueTask ApproveRenameAsync(CloudProviderRenameRequest request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref Approvals); + Approval.TrySetResult(request); + return ValueTask.FromResult(CloudProviderPolicyDecision.Allow); + } + + public ValueTask ApproveDeleteAsync(CloudProviderDeleteRequest request, CancellationToken cancellationToken) => + ValueTask.FromResult(CloudProviderPolicyDecision.Allow); + + public ValueTask OnCompletionAsync(CloudProviderCompletionNotification notification, CancellationToken cancellationToken) + { + if (notification.Kind == CloudProviderNotificationKind.RenameCompleted) + { + Interlocked.Increment(ref Completions); + Completion.TrySetResult(notification); + } + + return ValueTask.CompletedTask; + } + } + + private sealed class NamespaceProcess : IDisposable + { + private readonly Process _process; + private readonly EventWaitHandle _ready; + internal EventWaitHandle Go { get; } + internal EventWaitHandle Attempted { get; } + internal EventWaitHandle Completed { get; } + internal int ExitCode => _process.ExitCode; + + internal NamespaceProcess(string source, string target, string action) + { + string signal = "Local\\CfSharp-directory-race-" + Guid.NewGuid().ToString("N"); + _ready = new(false, EventResetMode.ManualReset, signal); + Go = new(false, EventResetMode.ManualReset, signal + "-go"); + Attempted = new(false, EventResetMode.ManualReset, signal + "-attempted"); + Completed = new(false, EventResetMode.ManualReset, signal + "-completed"); + ProcessStartInfo start = new("dotnet") { UseShellExecute = false, CreateNoWindow = true }; + foreach (string argument in new[] { DirectoryHarnessPath(), source, target, signal, action, "namespace-consumer" }) + { + start.ArgumentList.Add(argument); + } + + _process = Process.Start(start)!; + } + + internal async Task WaitForReadyAsync() + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); + while (!_ready.WaitOne(0)) + { + Assert.False(_process.HasExited); + await Task.Delay(10, timeout.Token); + } + } + + internal Task WaitForExitAsync() => _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(20)); + internal void WaitForExit() => Assert.True(_process.WaitForExit(20000)); + public void Dispose() + { + if (!_process.HasExited) + { + _process.Kill(entireProcessTree: true); + _process.WaitForExit(); + } + + _process.Dispose(); + _ready.Dispose(); + Go.Dispose(); + Attempted.Dispose(); + Completed.Dispose(); + } + } + + private static string DirectoryHarnessPath() + { + DirectoryInfo? repository = new(AppContext.BaseDirectory); + while (repository is not null && !File.Exists(Path.Combine(repository.FullName, "CfSharp.sln"))) + { + repository = repository.Parent; + } + + Assert.NotNull(repository); + string configuration = typeof(CloudDirectoryMoveTests).Assembly + .GetCustomAttributes(typeof(System.Reflection.AssemblyConfigurationAttribute), false) + .Cast().Single().Configuration; + string path = Path.Combine(repository!.FullName, "tests", "CfSharp.Storage.Sqlite.CrashHarness", "bin", configuration, + "net10.0-windows", "CfSharp.Storage.Sqlite.CrashHarness.dll"); + Assert.True(File.Exists(path)); + return path; + } +} diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs index b9cbf46..2fbc9ed 100644 --- a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs @@ -95,7 +95,7 @@ internal sealed class DirectoryMoveTestRoot(CloudFileSystem fileSystem, Director internal DirectoryMoveFaultStore Faults => factory.Store!; internal CloudPlaceholderIdentity DirectoryIdentity => directoryIdentity; - internal static async Task OpenAsync() + internal static async Task OpenAsync(ICloudFileContentProvider? provider = null) { string area = Path.Combine(Path.GetTempPath(), "CfSharp-directory-move-tests", Guid.NewGuid().ToString("N")); string rootPath = Path.Combine(area, "root"); @@ -106,7 +106,7 @@ internal static async Task OpenAsync() .WithRegistration(SyncRootRegistrationOptions.CreateBuilder("CfSharp Directory Move", "1.0-test") .WithProviderId(providerId).WithSyncRootIdentity(providerId.ToByteArray()) .WithHydrationPolicy(CloudHydrationPolicy.Progressive).WithPopulationPolicy(CloudPopulationPolicy.Partial) - .WithRootMarkedInSync().Build()).WithContentProvider(new EmptyContentProvider()).Build(); + .WithRootMarkedInSync().Build()).WithContentProvider(provider ?? new EmptyContentProvider()).Build(); bool registered = false; try { diff --git a/tests/CfSharp.Storage.Sqlite.CrashHarness/NamespaceConsumer.cs b/tests/CfSharp.Storage.Sqlite.CrashHarness/NamespaceConsumer.cs new file mode 100644 index 0000000..be435fd --- /dev/null +++ b/tests/CfSharp.Storage.Sqlite.CrashHarness/NamespaceConsumer.cs @@ -0,0 +1,38 @@ +internal static class NamespaceConsumer +{ + internal static int Run(string source, string target, string signal, string action) + { + using EventWaitHandle ready = EventWaitHandle.OpenExisting(signal); + using EventWaitHandle go = EventWaitHandle.OpenExisting(signal + "-go"); + using EventWaitHandle attempted = EventWaitHandle.OpenExisting(signal + "-attempted"); + using EventWaitHandle completed = EventWaitHandle.OpenExisting(signal + "-completed"); + ready.Set(); + if (!go.WaitOne(TimeSpan.FromSeconds(15))) + { + return 2; + } + + attempted.Set(); + try + { + switch (action) + { + case "rename": Directory.Move(source, target); break; + case "delete": File.Delete(source); break; + case "edit": File.WriteAllText(source, "independent edit"); break; + case "create": File.WriteAllText(target, "independent child"); break; + default: return 3; + } + + return 0; + } + catch (IOException exception) + { + return exception.HResult & 0xffff; + } + finally + { + completed.Set(); + } + } +} diff --git a/tests/CfSharp.Storage.Sqlite.CrashHarness/Program.cs b/tests/CfSharp.Storage.Sqlite.CrashHarness/Program.cs index 237ea52..9272012 100644 --- a/tests/CfSharp.Storage.Sqlite.CrashHarness/Program.cs +++ b/tests/CfSharp.Storage.Sqlite.CrashHarness/Program.cs @@ -1,6 +1,11 @@ using CfSharp; using CfSharp.Storage.Sqlite; +if (args.Length == 5 && args[4] == "namespace-consumer") +{ + return NamespaceConsumer.Run(args[0], args[1], args[2], args[3]); +} + if (args.Length == 3 && args[2] == "protected-deadline" && OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) { return await ProtectedConfirmationDeadline.RunAsync(args[0], args[1]); From 1a4fa5551bf9e2448e8a733fdccba4deca35fb15 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:36:19 +0800 Subject: [PATCH 15/40] test(items): verify directory move recovery across process crashes --- .../CloudDirectoryMoveCrashTests.cs | 89 +++++++++++++++++++ .../CloudDirectoryMoveTests.cs | 8 ++ .../DirectoryMoveCrash.cs | 89 +++++++++++++++++++ .../Program.cs | 5 ++ 4 files changed, 191 insertions(+) create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveCrashTests.cs create mode 100644 tests/CfSharp.Storage.Sqlite.CrashHarness/DirectoryMoveCrash.cs diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCrashTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCrashTests.cs new file mode 100644 index 0000000..074625c --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCrashTests.cs @@ -0,0 +1,89 @@ +using System.Diagnostics; +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData("directory-prepared", 91)] + [InlineData("directory-native", 92)] + [InlineData("directory-before-projection", 93)] + [InlineData("directory-after-projection", 94)] + public async Task AbruptDirectoryMoveProcessExitRecoversOriginalProofAndAllDurableFacts(string mode, int exitCode) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudOperationJournalEntry pending = await SeedPendingChildAsync(root); + Guid tombstoneId = Guid.NewGuid(); + await using (ICloudStateTransaction seed = await root.Store.BeginTransactionAsync()) + { + await seed.Items.UpsertAsync(new(tombstoneId, "deleted", "Docs\\gone.txt", CloudItemKind.File, + "deleted-revision", 99, true, DateTimeOffset.UtcNow)); + await seed.CommitAsync(); + } + + await root.FileSystem.DisposeAsync(); + ProcessStartInfo start = new("dotnet") { UseShellExecute = false, CreateNoWindow = true }; + foreach (string argument in new[] { DirectoryHarnessPath(), root.DatabasePath, root.RootPath, mode }) + { + start.ArgumentList.Add(argument); + } + + using Process child = Process.Start(start)!; + try + { + await child.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(20)); + Assert.Equal(exitCode, child.ExitCode); + Assert.True(File.Exists(root.DatabasePath + "." + mode + ".ready")); + } + finally + { + if (!child.HasExited) + { + child.Kill(entireProcessTree: true); + await child.WaitForExitAsync(); + } + } + + CloudDirectoryMoveProof proof = CloudDirectoryMoveProof.Decode(await File.ReadAllBytesAsync(root.DatabasePath + ".directory-proof")); + await root.RestartAsync(); + bool committed = mode == "directory-after-projection"; + string initialDirectory = committed ? "Moved" : "Docs"; + await AssertProjectedChildAsync(root, initialDirectory, pending); + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + Assert.Equal(committed, await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId)) is not null); + Assert.Equal(proof.Encode(), (await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ProofName(proof.ProofId)))!.Value.ToArray()); + Assert.Equal(initialDirectory + "\\gone.txt", (await verify.Items.GetByItemIdAsync(tombstoneId))!.RelativePath); + } + + CloudDirectory original = root.FileSystem.GetDirectory("Docs"); + if (mode == "directory-prepared") + { + CloudDirectoryMoveReconciliationResult notMoved = await original.ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.NotMoved, notMoved.Outcome); + Assert.False(notMoved.NativeMoveObserved || notMoved.DurableProjectionCommitted); + Directory.Move(original.FullPath, Path.Combine(root.RootPath, "Moved")); + } + + Assert.False(Directory.Exists(original.FullPath)); + Assert.Equal(proof.ExpectedBinding, (await root.FileSystem.GetDirectory("Moved").InspectAsync()).LocalBinding); + CloudDirectoryMoveReconciliationResult recovered = await original.ReconcileMoveAsync(proof); + Assert.Equal(committed ? CloudDirectoryMoveReconciliationOutcome.AlreadyProjected : CloudDirectoryMoveReconciliationOutcome.Projected, recovered.Outcome); + Assert.True(recovered.NativeMoveObserved && recovered.DurableProjectionCommitted); + Assert.False(recovered.RequiresFullRescan); + await AssertProjectedChildAsync(root, "Moved", pending); + await root.RestartAsync(); + CloudDirectoryMoveReconciliationResult duplicate = await root.FileSystem.GetDirectory("Docs").ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.AlreadyProjected, duplicate.Outcome); + Assert.Equal(0, duplicate.DurableStateEntriesUpdated); + await using ICloudStateTransaction final = await root.Store.BeginTransactionAsync(); + CloudItemState deleted = (await final.Items.GetByItemIdAsync(tombstoneId))!; + Assert.True(deleted.IsTombstone); + Assert.Equal("Moved\\gone.txt", deleted.RelativePath); + Assert.Equal("deleted-revision", deleted.RemoteRevision); + Assert.Equal(99, deleted.LocalFileId); + Assert.Single(await final.Operations.ListAsync(100)); + } +} diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs index 2fbc9ed..d4e4329 100644 --- a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs @@ -95,6 +95,14 @@ internal sealed class DirectoryMoveTestRoot(CloudFileSystem fileSystem, Director internal DirectoryMoveFaultStore Faults => factory.Store!; internal CloudPlaceholderIdentity DirectoryIdentity => directoryIdentity; + internal async Task RestartAsync() + { + string root = RootPath; + await fileSystem.DisposeAsync(); + fileSystem = CloudFileSystem.CreateBuilder(root).WithStateStore(factory).WithContentProvider(new EmptyContentProvider()).Build(); + await fileSystem.StartAsync(); + } + internal static async Task OpenAsync(ICloudFileContentProvider? provider = null) { string area = Path.Combine(Path.GetTempPath(), "CfSharp-directory-move-tests", Guid.NewGuid().ToString("N")); diff --git a/tests/CfSharp.Storage.Sqlite.CrashHarness/DirectoryMoveCrash.cs b/tests/CfSharp.Storage.Sqlite.CrashHarness/DirectoryMoveCrash.cs new file mode 100644 index 0000000..c9421ca --- /dev/null +++ b/tests/CfSharp.Storage.Sqlite.CrashHarness/DirectoryMoveCrash.cs @@ -0,0 +1,89 @@ +using System.Runtime.Versioning; + +using CfSharp; +using CfSharp.Storage.Sqlite; + +[SupportedOSPlatform("windows10.0.16299")] +internal static class DirectoryMoveCrash +{ + internal static async Task RunAsync(string database, string root, string mode) + { + if (mode is not ("directory-prepared" or "directory-native" or "directory-before-projection" or "directory-after-projection")) + { + return 2; + } + + CrashFactory factory = new(database, mode); + await using CloudFileSystem system = CloudFileSystem.CreateBuilder(root).WithStateStore(factory) + .WithContentProvider(new NoFetchProvider()).Build(); + await system.StartAsync(); + CloudDirectory source = system.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(system.Root, "Moved"); + // The application's retained original proof is outside both the root and state database. + File.WriteAllBytes(database + ".directory-proof", proof.Encode()); + File.WriteAllText(database + "." + mode + ".ready", proof.ProofId.ToString("N")); + if (mode == "directory-prepared") + { + Environment.Exit(91); + } + + Directory.Move(source.FullPath, Path.Combine(root, "Moved")); + if (mode == "directory-native") + { + Environment.Exit(92); + } + + factory.Armed = true; + await source.ReconcileMoveAsync(proof); + return 3; // Missing a selected abrupt-exit point is a failing harness result. + } + + private sealed class NoFetchProvider : ICloudFileContentProvider + { + public ValueTask OpenReadAsync(CloudFileFetchRequest request, CancellationToken cancellationToken) => + ValueTask.FromException(new InvalidOperationException("Directory recovery must not hydrate content.")); + } + + private sealed class CrashFactory(string path, string mode) : ICloudStateStoreFactory + { + internal bool Armed { get; set; } + private string Mode => mode; + public async ValueTask OpenAsync(CloudStateStoreContext context, CancellationToken cancellationToken = default) => + new Store(await new SqliteCloudStateStoreFactory(path).OpenAsync(context, cancellationToken), this); + + private sealed class Store(ICloudStateStore inner, CrashFactory owner) : ICloudStateStore + { + public async ValueTask BeginTransactionAsync(CancellationToken cancellationToken = default) => + new Transaction(await inner.BeginTransactionAsync(cancellationToken), owner); + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + + private sealed class Transaction(ICloudStateTransaction inner, CrashFactory owner) : ICloudStateTransaction + { + public ICloudItemStateRepository Items => inner.Items; + public ICloudCheckpointRepository Checkpoints => inner.Checkpoints; + public ICloudOperationJournal Operations => inner.Operations; + public ICloudConflictRepository Conflicts => inner.Conflicts; + public ICloudRemoteBatchRepository RemoteBatches => inner.RemoteBatches; + public ICloudEchoSuppressionRepository EchoSuppressions => inner.EchoSuppressions; + public async ValueTask CommitAsync(CancellationToken cancellationToken = default) + { + if (owner.Armed && owner.Mode == "directory-before-projection") + { + // Exit without finally/disposal while the real SQLite transaction contains + // all projected paths and its receipt; WAL recovery must discard them together. + Environment.Exit(93); + } + + await inner.CommitAsync(cancellationToken); + if (owner.Armed && owner.Mode == "directory-after-projection") + { + Environment.Exit(94); + } + } + + public ValueTask RollbackAsync(CancellationToken cancellationToken = default) => inner.RollbackAsync(cancellationToken); + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + } +} diff --git a/tests/CfSharp.Storage.Sqlite.CrashHarness/Program.cs b/tests/CfSharp.Storage.Sqlite.CrashHarness/Program.cs index 9272012..76b080a 100644 --- a/tests/CfSharp.Storage.Sqlite.CrashHarness/Program.cs +++ b/tests/CfSharp.Storage.Sqlite.CrashHarness/Program.cs @@ -6,6 +6,11 @@ return NamespaceConsumer.Run(args[0], args[1], args[2], args[3]); } +if (args.Length == 3 && args[2].StartsWith("directory-", StringComparison.Ordinal) && OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) +{ + return await DirectoryMoveCrash.RunAsync(args[0], args[1], args[2]); +} + if (args.Length == 3 && args[2] == "protected-deadline" && OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) { return await ProtectedConfirmationDeadline.RunAsync(args[0], args[1]); From 0804a7ea2005223bc160b479f24b4ea55c4090be Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:45:01 +0800 Subject: [PATCH 16/40] test(compat): preserve preview3 state and public behavior --- .../CfSharp.IntegrationTests.csproj | 5 ++ .../CloudDirectoryMoveCompatibilityTests.cs | 48 ++++++++++++ .../CloudDirectoryMoveTests.cs | 7 +- .../CfSharp.Storage.Sqlite.Tests.csproj | 4 + .../Fixtures/preview3-state.zip | Bin 0 -> 2053 bytes .../PreviewThreeCompatibilityTests.cs | 55 +++++++++++++ .../PreviewThreeStateFixture.cs | 73 ++++++++++++++++++ 7 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveCompatibilityTests.cs create mode 100644 tests/CfSharp.Storage.Sqlite.Tests/Fixtures/preview3-state.zip create mode 100644 tests/CfSharp.Storage.Sqlite.Tests/PreviewThreeCompatibilityTests.cs create mode 100644 tests/CfSharp.Storage.Sqlite.Tests/PreviewThreeStateFixture.cs diff --git a/tests/CfSharp.IntegrationTests/CfSharp.IntegrationTests.csproj b/tests/CfSharp.IntegrationTests/CfSharp.IntegrationTests.csproj index adb9612..72aeccd 100644 --- a/tests/CfSharp.IntegrationTests/CfSharp.IntegrationTests.csproj +++ b/tests/CfSharp.IntegrationTests/CfSharp.IntegrationTests.csproj @@ -23,4 +23,9 @@ + + + + + diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCompatibilityTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCompatibilityTests.cs new file mode 100644 index 0000000..d614368 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCompatibilityTests.cs @@ -0,0 +1,48 @@ +using System.Runtime.Versioning; + +using CfSharp.Tests.Persistence; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Fact] + public async Task PublishedPreviewThreeStateSurvivesDirectoryFeedProjectionProofReplayAndRuntimeRestart() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(legacyState: true); + await using (ICloudStateTransaction before = await root.Store.BeginTransactionAsync()) + { + await PreviewThreeStateFixture.AssertRetainedAsync(before); + } + + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(root.FileSystem.Root, "Moved"); + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Moved")); + await ProcessRenameAsync(root, "Docs", "Moved"); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.AlreadyProjected, (await source.ReconcileMoveAsync(proof)).Outcome); + await root.RestartAsync(); + CloudItemMoveResult replay = await root.FileSystem.GetDirectory("Docs").MoveToAsync(root.FileSystem.Root, "Moved", new CloudMoveOptions(proof)); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.AlreadyProjected, replay.DirectoryReconciliation!.Outcome); + Assert.False(replay.DirectoryReconciliation.RequiresFullRescan); + await using (ICloudStateTransaction after = await root.Store.BeginTransactionAsync()) + { + await PreviewThreeStateFixture.AssertRetainedAsync(after, "Moved"); + Assert.NotNull(await after.Items.GetByRelativePathAsync("Moved\\child.txt")); + } + + await using CloudLocalChangeFeed feed = CloudLocalChangeFeed.CreateForTesting(root.RootPath, root.Store, new(), new FeedSource()); + await feed.StartAsync(); + CloudLocalChangeScan scan = await feed.BeginScanAsync(); + Assert.False(scan.RequiresFullRescan); + CloudLocalChangeJournalException foreign = await Assert.ThrowsAsync(() => feed.ReadPageAsync(scan, 0, 1).AsTask()); + Assert.Equal(PreviewThreeStateFixture.OperationId, foreign.OperationId); + CloudLocalChange move = Assert.Single((await feed.ReadPageAsync(scan, 1, 1)).Changes); + Assert.Equal(CloudLocalChangeKind.Move, move.Kind); + Assert.Equal("Docs", move.PreviousRelativePath); + Assert.Equal("Moved", move.RelativePath); + await using ICloudStateTransaction retained = await root.Store.BeginTransactionAsync(); + Assert.NotNull(await retained.Operations.GetAsync(PreviewThreeStateFixture.OperationId)); + Assert.NotNull(await retained.Operations.GetAsync(move.OperationId)); + } +} diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs index d4e4329..9f99920 100644 --- a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs @@ -1,6 +1,7 @@ using System.Runtime.Versioning; using CfSharp.Storage.Sqlite; +using CfSharp.Tests.Persistence; namespace CfSharp.IntegrationTests; @@ -103,11 +104,15 @@ internal async Task RestartAsync() await fileSystem.StartAsync(); } - internal static async Task OpenAsync(ICloudFileContentProvider? provider = null) + internal static async Task OpenAsync(ICloudFileContentProvider? provider = null, bool legacyState = false) { string area = Path.Combine(Path.GetTempPath(), "CfSharp-directory-move-tests", Guid.NewGuid().ToString("N")); string rootPath = Path.Combine(area, "root"); Directory.CreateDirectory(Path.Combine(rootPath, "Ordinary")); + if (legacyState) + { + PreviewThreeStateFixture.Extract(Path.Combine(area, "state.db"), rootPath); + } Guid providerId = Guid.NewGuid(); CapturingFactory factory = new(new SqliteCloudStateStoreFactory(Path.Combine(area, "state.db"))); CloudFileSystem fileSystem = CloudFileSystem.CreateBuilder(rootPath).WithStateStore(factory) diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CfSharp.Storage.Sqlite.Tests.csproj b/tests/CfSharp.Storage.Sqlite.Tests/CfSharp.Storage.Sqlite.Tests.csproj index 3b8b907..058148b 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CfSharp.Storage.Sqlite.Tests.csproj +++ b/tests/CfSharp.Storage.Sqlite.Tests/CfSharp.Storage.Sqlite.Tests.csproj @@ -26,4 +26,8 @@ + + + + diff --git a/tests/CfSharp.Storage.Sqlite.Tests/Fixtures/preview3-state.zip b/tests/CfSharp.Storage.Sqlite.Tests/Fixtures/preview3-state.zip new file mode 100644 index 0000000000000000000000000000000000000000..2c62b678850e18832cfc1fee29df63e60673aa1d GIT binary patch literal 2053 zcmZ`)c~sI_8^)T3%F3xEGc#u@HTNW!6wPUlS&C%aSmr{O3%G%jxQp6XDyB{8SeBYu zifMw7ONfYBfy|_2h)Zs%;6f%TVu9dq98Z4d`{(=P-h0k_p8LGd^PGF`dkVhx2Tjl# z(0UM}>A3IpV8p#_FbD*?qy|!7KCn@NcywUQA@hLCp9guE{%*CF$TKwkv$K@(WysK z!VG6_qVGVZMzo~(Z4C2xl6>!u2o>7h&i;BW7%ogmOk7@?uD3rVV+J?;hboCr$6qov z&UM@Ss409Wv2?$1_<e<}hP^|-QK`d9Yr`_Raku;|}(0qHl)aRSul zwj4E|PypLU5G)lmL+3s{@;F1-y^YJe6ESL1psCf)9muX-y5j5j6ege@ChK|Ae03H1 zhQ|1hyEjw?fx(_kaozx}rZR%NaO~b_yT}ZO=2#K0*L@|HQ#~(O>S?b<6k9&YPTN{u zm|bR-Z`<9YLpP^Hu%6e}p}7NhZ9UzqYN$=M;X${^B$+ZTey*sdrh}^Q1}5w0gyt9? zimVzbK~e+`Ba2bJtWMLwb4^1Vc8MknP&PNUnghX}AM=V8eH!F4S;AQ{*FCgZzo8Ks z>6VK5)4?BHIFTgW`I-%=)A|g|o}>8=Cx0CtWX#W@hx#u`;LQt5RISoA0A zELk3V#C_05tvvwhYNdGU6Hh+{8()j7a*Wo5U>LL0JPfyUo`yZ1<1P8%W4~^v>Gnmh zH+6rrRkG|wU(u%sPLkHxJACCyS*B=4-U=gzkPNBR%Vcv%Pt46kt3EM1pT1D??n0_d zl`J6J3?~?Xs{cKX$F0S1+8U9TJqY^Y{-C@o1`Ml$_gU(d+v*PjDx!N>pR8qbdUbJp z-JIj&q1ec1)9T>oVvc7oUfB8qn(GKVpu67bh}E7@>)97Uxf{~sb>LNdXDEy zdE#P8e6S$Zx9a!Z^rWnikB+%TN140kpWbNMJpC7FO4+hStP;-^cGp;hUWm2k-L~VD z_Cpp?K?ZIYsj3!Z-QK9QL-iT))w@5@;)!>-ODKmjOu(WbG_l=V{_V?a=~EMQQiz)h zW%{fWkL*i6w5y7Hil>!%lF#(%T)~~QS~8u#5lXWBunc%-6WFWA=~HC*zrs*g-?`pGZG zk5X4)8e*h+d6g9u6#^7Wns|l6vuM>gLKbK9L9xYo3XM3qw}I8se>G8k+Sw|!zVpOV zyZL+@x$BLWNpi}}xSxF)WpJ9|c$H|_kkQf7CS^Kb?1`3*Js2sy5o)Epd(+JCa3%fH zES}$R1-*cpsP6!j{8uH}Om5wbnV=5^F-NloMXJy2 zsq4P|>V-*+tvY^M+w@Gw7mCJ8W-xk2i>pR2%f6>HhH z-m2|w^gHh8YB0!!10YxL!#`exlA?iwEiBL}xY`;`(Ep#?EA0Q>{%4Axg0EY732GoU Nkliw%{@-B_`UlP3|7idK literal 0 HcmV?d00001 diff --git a/tests/CfSharp.Storage.Sqlite.Tests/PreviewThreeCompatibilityTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/PreviewThreeCompatibilityTests.cs new file mode 100644 index 0000000..e94d5c9 --- /dev/null +++ b/tests/CfSharp.Storage.Sqlite.Tests/PreviewThreeCompatibilityTests.cs @@ -0,0 +1,55 @@ +using CfSharp.Tests.Persistence; + +using Microsoft.Data.Sqlite; + +namespace CfSharp.Storage.Sqlite.Tests; + +public sealed class PreviewThreeCompatibilityTests +{ + [Fact] + public async Task PublishedPreviewThreeDatabaseSupportsBoundedPagingAndCompleteRestart() + { + string area = Path.Combine(Path.GetTempPath(), "CfSharp-preview3-compatibility", Guid.NewGuid().ToString("N")); + string root = Path.Combine(area, "root"); + string database = Path.Combine(area, "state.db"); + Directory.CreateDirectory(root); + PreviewThreeStateFixture.Extract(database, root); + try + { + await using (SqliteConnection legacy = new($"Data Source={database};Mode=ReadOnly;Pooling=False")) + { + await legacy.OpenAsync(); + using SqliteCommand command = legacy.CreateCommand(); + command.CommandText = "SELECT version FROM cfsharp_schema WHERE singleton=1;"; + Assert.Equal(5L, await command.ExecuteScalarAsync()); + } + + Guid nextId = Guid.NewGuid(); + await using (ICloudStateStore store = await new SqliteCloudStateStoreFactory(database).OpenAsync(new(root))) + await using (ICloudStateTransaction transaction = await store.BeginTransactionAsync()) + { + await PreviewThreeStateFixture.AssertRetainedAsync(transaction); + ICloudOperationJournalPaging paging = Assert.IsAssignableFrom(transaction.Operations); + long boundary = await paging.GetHighWaterSequenceAsync(); + Assert.Equal(1, boundary); + CloudOperationJournalPage page = await paging.ReadPageAsync(0, boundary, 1); + Assert.Equal(PreviewThreeStateFixture.OperationId, Assert.Single(page.Operations).OperationId); + await transaction.Operations.EnqueueAsync(new(nextId, CloudStateOperationKind.MetadataUpdate, null, new byte[] { 11 }, DateTimeOffset.UtcNow)); + Assert.Empty((await paging.ReadPageAsync(boundary, boundary, 1)).Operations); + await transaction.Checkpoints.UpsertAsync(new("compatibility/new-runtime", new byte[] { 12 }, DateTimeOffset.UtcNow)); + await transaction.CommitAsync(); + } + + await using ICloudStateStore reopened = await new SqliteCloudStateStoreFactory(database).OpenAsync(new(root)); + await using ICloudStateTransaction verify = await reopened.BeginTransactionAsync(); + await PreviewThreeStateFixture.AssertRetainedAsync(verify); + Assert.Equal(2, (await verify.Operations.GetAsync(nextId))!.Sequence); + Assert.Equal(new byte[] { 12 }, (await verify.Checkpoints.GetAsync("compatibility/new-runtime"))!.Value.ToArray()); + } + finally + { + SqliteConnection.ClearAllPools(); + Directory.Delete(area, recursive: true); + } + } +} diff --git a/tests/CfSharp.Storage.Sqlite.Tests/PreviewThreeStateFixture.cs b/tests/CfSharp.Storage.Sqlite.Tests/PreviewThreeStateFixture.cs new file mode 100644 index 0000000..613f0f5 --- /dev/null +++ b/tests/CfSharp.Storage.Sqlite.Tests/PreviewThreeStateFixture.cs @@ -0,0 +1,73 @@ +using System.IO.Compression; + +using Microsoft.Data.Sqlite; + +namespace CfSharp.Tests.Persistence; + +// Generated with the three public 0.1.0-preview.3 packages from NuGet, source commit +// b65b6b7a81c411e8af84f421e4259825e91536d6. SQLite package SHA-256: +// EF7E7FBA235125963365E5FBD031CD99660BECE60C760920957587B5CAF86189. +// Its real schema-5 database was checkpointed after closing the old store; no schema +// number was rewritten. Only the 2 KiB compressed database is a checked-in test asset. +internal static class PreviewThreeStateFixture +{ + internal static readonly Guid ItemId = Guid.Parse("11111111-2222-3333-4444-555555555555"); + internal static readonly Guid OperationId = Guid.Parse("22222222-2222-3333-4444-555555555555"); + private static readonly Guid ConflictId = Guid.Parse("33333333-2222-3333-4444-555555555555"); + private static readonly Guid SuppressionId = Guid.Parse("44444444-2222-3333-4444-555555555555"); + private static readonly DateTimeOffset Timestamp = new(2026, 10, 1, 0, 0, 0, TimeSpan.Zero); + + internal static void Extract(string database, string syncRoot) + { + Directory.CreateDirectory(Path.GetDirectoryName(database)!); + using Stream resource = typeof(PreviewThreeStateFixture).Assembly.GetManifestResourceStream("CfSharp.PreviewThreeStateFixture.zip")!; + using ZipArchive archive = new(resource, ZipArchiveMode.Read); + using Stream entry = archive.GetEntry("preview3.db")!.Open(); + using (FileStream destination = File.Create(database)) + { + entry.CopyTo(destination); + } + + // Rehome only this disposable fixture's machine-specific root path. Preserve the + // genuine schema version and every legacy repository value; production databases + // remain bound to their original root and must never be relocated this way. + using SqliteConnection connection = new($"Data Source={database};Pooling=False"); + connection.Open(); + using SqliteCommand command = connection.CreateCommand(); + command.CommandText = "UPDATE cfsharp_schema SET sync_root_path=$root WHERE singleton=1;"; + command.Parameters.AddWithValue("$root", Path.GetFullPath(syncRoot)); + Assert.Equal(1, command.ExecuteNonQuery()); + } + + internal static async Task AssertRetainedAsync(ICloudStateTransaction transaction, string directory = "Docs") + { + CloudItemState item = (await transaction.Items.GetByItemIdAsync(ItemId))!; + Assert.Equal(directory + "\\deleted.txt", item.RelativePath); + Assert.Equal("legacy-remote", item.RemoteId); + Assert.Equal("legacy-revision", item.RemoteRevision); + Assert.Equal(321, item.LocalFileId); + Assert.True(item.IsTombstone); + CloudStateCheckpoint checkpoint = (await transaction.Checkpoints.GetAsync("legacy/checkpoint"))!; + Assert.Equal(new byte[] { 1, 2 }, checkpoint.Value.ToArray()); + Assert.Equal(Timestamp, checkpoint.UpdatedAt); + CloudOperationJournalEntry operation = (await transaction.Operations.GetAsync(OperationId))!; + Assert.Equal(1, operation.Sequence); + Assert.Equal(ItemId, operation.ItemId); + Assert.Equal(new byte[] { 3, 4 }, operation.Payload.ToArray()); + Assert.Equal(Timestamp, operation.CreatedAt); + Assert.Equal(3, operation.AttemptCount); + Assert.Equal(Timestamp.AddDays(90), operation.RetryAfter); + Assert.Equal(new byte[] { 5, 6 }, (await transaction.Conflicts.GetAsync(ConflictId))!.Payload.ToArray()); + CloudRemoteBatchState batch = (await transaction.RemoteBatches.GetAsync("legacy-batch"))!; + Assert.Equal(CloudRemoteBatchStatus.Applying, batch.Status); + Assert.Equal(new byte[] { 7 }, batch.Cursor.ToArray()); + Assert.Equal(new byte[] { 8 }, batch.Payload.ToArray()); + Assert.Equal(new byte[] { 9 }, batch.Fingerprint.ToArray()); + Assert.Equal("legacy-change", batch.LastAppliedChangeId); + CloudEchoSuppressionState suppression = (await transaction.EchoSuppressions.GetAsync(SuppressionId))!; + Assert.Equal("Old\\deleted.txt", suppression.PreviousRelativePath); + Assert.Equal("Docs\\deleted.txt", suppression.RelativePath); + Assert.Equal(2, suppression.RemainingObservations); + Assert.Equal(new byte[] { 10 }, suppression.Payload.ToArray()); + } +} From b99c5e52e63bebb3cf04d342555380dbab411bd7 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:58:08 +0800 Subject: [PATCH 17/40] docs: explain bounded scans and proven directory recovery --- README.md | 5 ++++ docs/local-change-feed.md | 14 +++++++--- docs/placeholders.md | 52 ++++++++++++++++++++++++++++++++++++++ docs/state-store-sqlite.md | 9 +++++++ docs/troubleshooting.md | 22 ++++++++++++++++ 5 files changed, 99 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 84284fd..8203cdc 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,11 @@ The dependency direction is one way: `CfSharp.Native` is the interop layer, `CfS the safe domain API, and the SQLite package depends on `CfSharp`. Remote transport, authentication, content bytes, and business conflict policy remain application-owned. +The source API supports [finite typed scans of pending local changes](docs/local-change-feed.md) +without acknowledging a deferred head, and [verified directory move recovery](docs/placeholders.md) +that preserves subtree identities, current revisions, and pending journal operations. Applications +retain their own dispatch policy and durable move intents. + ## Quick start ### Requirements diff --git a/docs/local-change-feed.md b/docs/local-change-feed.md index c56d04c..5242ccd 100644 --- a/docs/local-change-feed.md +++ b/docs/local-change-feed.md @@ -47,8 +47,10 @@ while (true) break; } - await DispatchEligibleChangesAsync(page.Changes, cancellationToken); - // ACK only operation IDs whose meaning the application has durably accepted. + IReadOnlyList acceptedIds = await DispatchEligibleChangesAsync( + page.Changes, cancellationToken); + // This application method returns only IDs whose meaning it durably accepted. + await feed.AcknowledgeAsync(acceptedIds, cancellationToken); after = page.LastScannedSequence; if (!page.HasMore) { @@ -69,6 +71,10 @@ stores throw `NotSupportedException` for scans and retain the existing batch API journal payloads throw `CloudLocalChangeJournalException` with the original operation ID and sequence, leaving the row pending. Consumers do not need private payload parsing or a copied queue. Scheduling across roots and preserving dependencies between changes remain application concerns. +Keep a dependency view across pages before dispatching dependent moves/deletes. A deferred root +does not have to be ACKed to reach an independent later root. Completing this finite scan does not +accept deferred operations: begin the next cycle at zero, preserving those original IDs. Later writes +belong to the next captured boundary. Multiple consumers must coordinate dispatch and ACK ownership. ## Rescan is a normal state @@ -87,7 +93,7 @@ after the durable marker is cleared. Begin a new scan after recovery. If a new l `AcknowledgeFullRescanAsync`, it either rejects acknowledgement with `InvalidOperationException` or remains observable as a new rescan condition; finish reconciliation before retrying. -## Provider echoes +## Directory rename projection Directory renames use retained library-captured native bindings and historical membership to validate the target. Paths of the complete known subtree, its immutable move receipt, the new @@ -107,6 +113,8 @@ For an existing partial projection, use the original source directory's `MoveToA prepared-proof recovery described in [placeholders](placeholders.md). A matching path or remote ID alone cannot establish historical native ownership. +## Provider echoes + Provider-originated writes can be wrapped by `SuppressProviderEchoAsync` so they do not become uploads. Hydration, pinning, and availability transitions are not local upload operations by themselves. diff --git a/docs/placeholders.md b/docs/placeholders.md index 8163d57..844252c 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -53,6 +53,58 @@ directory binding and known membership into an independent immutable preparation manufactures historical evidence from a current target. Legacy directories without either source of provenance still require explicit preparation while at the original source. +### Retain and recover an external move + +Keep the original source reference, exact destination spelling, and original encoded proof in +application-owned durable intent storage outside the sync root. Before an external consumer is +allowed to rename `Docs`: + +```csharp +CloudDirectory original = fileSystem.GetDirectory("Docs"); +CloudDirectoryMoveProof proof = await original.PrepareMoveAsync( + fileSystem.Root, "Renamed", cancellationToken); +await File.WriteAllBytesAsync(applicationProofPath, proof.Encode(), cancellationToken); +// The application can now authorize its external consumer to perform that exact rename. +``` + +After the external consumer finishes, or after restarting against the same store: + +```csharp +CloudDirectoryMoveProof retained = CloudDirectoryMoveProof.Decode( + await File.ReadAllBytesAsync(applicationProofPath, cancellationToken)); +CloudDirectoryMoveReconciliationResult result = await fileSystem.GetDirectory("Docs") + .ReconcileMoveAsync(retained, cancellationToken); + +if (result.RequiresFullRescan) +{ + await ReconcileEntireSyncRootAsync(fileSystem, cancellationToken); +} +// Inspect Outcome and native/committed facts before advancing the application intent. +// Reconciliation does not ACK local operations or accept a remote mutation. +``` + +`applicationProofPath` is an example application intent file, not a CfSharp state-store setting. +Persist it before allowing the native move; do not regenerate a missing original proof from the +destination's present identity. An existing partial projection can also use the original source's +`MoveToAsync` when CfSharp retained sufficient pre-rename provenance. Otherwise perform ownership +reconciliation without relaxing target guards. + +| Result | Application action | +| --- | --- | +| `NotMoved` | The proven source remains. Inspect the rescan flag before authorizing the intended move. | +| `Projected`, `AlreadyProjected` | Native movement and durable paths were validated; honor any remaining rescan requirement. | +| `NativeObservedProjectionPending` | Preserve the proof and retry after the original store failure is resolved. | +| `Conflict`, `NotApplicable` | Stop this intent and reconcile ownership; do not overwrite the target or invent provenance. | +| `Busy`, `Canceled`, `Failed` | Inspect Stage, Error, and both native/committed facts before retrying the same intent. | + +A post-commit race can have `DurableProjectionCommitted = true` with a conflict/rescan outcome. +`NativeHResult` is nullable because a logical conflict or store exception is not a failed native +call. Log bounded identifiers, stage, outcome, and original error details as appropriate; avoid +dumping complete opaque identities or arbitrary user paths into routine diagnostics. +Proof version 1 bounds encoded paths to 32 KiB of UTF-8, identities to the native 4 KiB limit, +and known-member manifests to 64 MiB. Unknown protocol versions fail closed. Preparation is +metadata-only and does not freeze child content, identity writes, or later namespace moves. + Placeholder operations use immutable, kind-specific specifications rather than exposing native unions and flag combinations directly to application code. diff --git a/docs/state-store-sqlite.md b/docs/state-store-sqlite.md index 9cc2ba5..1287797 100644 --- a/docs/state-store-sqlite.md +++ b/docs/state-store-sqlite.md @@ -62,3 +62,12 @@ Custom stores must enforce the same protocol boundary when opening persistent st runtimes. Preserve CfSharp-owned checkpoint names and values transactionally, including immutable preparations and receipts; never treat them as disposable caches or infer native ownership from an item path. The schema fence changes protocol compatibility, not the replaceable core interfaces. + +Upgrading a preview.3 schema-5 database retains all six repositories, original journal sequences +and retry fields, remote cursors/fingerprints, echo counts, and tombstones. The new directory +protocol is used only after that durable schema fence commits. Reopening an upgraded database +with an older package is rejected; do not rewrite its version to force a downgrade. +Custom stores keep the existing interfaces and may optionally implement journal paging. They must +atomically retain directory preparation, membership, projection receipt, item rows, and feed +checkpoint/journal updates when the core coordinates a move. Never inspect or overwrite CfSharp's +private namespace checkpoint payloads to fabricate recovery provenance. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 3b8acbe..65745a1 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -31,6 +31,28 @@ Replay is safe when the same immutable batch or callback is submitted again. Ver checkpoint and transaction commit rather than relying on an in-memory token. Duplicate attempts should converge to one namespace result; do not delete the state database to suppress a retry. +## A deferred journal head hides later changes + +Use `BeginScanAsync` and `ReadPageAsync` with the returned finite boundary. Keep deferred IDs +pending, carry dependencies across pages, and ACK only accepted IDs. `NotSupportedException` +means the custom journal lacks optional paging; add that capability rather than guessing an +unbounded read or rotating the queue. A `CloudLocalChangeJournalException` identifies a row whose +payload is not an official local change. Leave it pending and resolve it through its originating +component; consumers should not decode private payloads to bypass this error. + +## A directory moved but durable child paths are still old + +Retry the original source/destination through `MoveToAsync`, or decode and reconcile the original +prepared proof against the same store. Matching remote IDs, paths, or copied placeholder identity +do not prove that the target is the original native directory. Preserve the target and pending +journal on `Conflict`/`NotApplicable`; complete ownership reconciliation before resuming dispatch. + +Report Outcome, Stage, NativeMoveObserved, DurableProjectionCommitted, RequiresFullRescan, proof ID, +and the original Error. A null NativeHResult is expected for logical/store failures. A conflict after +commit can accurately report `DurableProjectionCommitted = true`: the native namespace changed +again after projection. Preserve the receipt and original proof rather than claiming rollback. +Full-rescan acknowledgement clears the fence but does not ACK the unresolved formal move. + ## Database locking or corruption concerns Keep one active owner per sync-root database, allow the bounded lock wait to expire, and preserve From 90df9be8b7e16667b6a0ad146721f77bc513d01a Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:11:37 +0800 Subject: [PATCH 18/40] build(release): verify preview capabilities in isolated consumers --- .github/workflows/ci.yml | 22 ++++ eng/prepare-preview.ps1 | 222 +++++++++++++++++++++++++++++++++++---- 2 files changed, 224 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da4bd85..76d10ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,21 @@ jobs: - name: Pack SQLite state provider run: dotnet pack src/CfSharp.Storage.Sqlite/CfSharp.Storage.Sqlite.csproj --configuration Release --no-build --output artifacts/packages -p:Version=0.0.0-ci.${{ github.run_number }} -p:PackageVersion=0.0.0-ci.${{ github.run_number }} + - name: Verify isolated package capabilities on x64 + shell: pwsh + run: ./eng/prepare-preview.ps1 -Version '0.0.0-preview.${{ github.run_number }}' -Channel Preview -OutputDirectory artifacts/package-capabilities + + - name: Upload x64 package capability evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: package-capabilities-x64 + path: | + artifacts/package-capabilities/**/manifest.json + artifacts/package-capabilities/**/consumer-smoke/packages.lock.json + artifacts/package-capabilities/**/packages/* + if-no-files-found: ignore + - name: Upload test results if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -146,6 +161,10 @@ jobs: - name: Test ARM64 solution run: dotnet test CfSharp.sln --configuration Release --no-build --no-restore --disable-build-servers -m:1 --filter "Category!=AbiProbe&Category!=ApiBaseline&Category!=LongSoak" --logger "trx;LogFileName=arm64-test-results.trx" --blame-hang-timeout 2m + - name: Verify isolated package capabilities on ARM64 + shell: pwsh + run: ./eng/prepare-preview.ps1 -Version '0.0.0-preview.${{ github.run_number }}' -Channel Preview -OutputDirectory artifacts/package-capabilities + - name: Upload ARM64 test results if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -154,6 +173,9 @@ jobs: path: | **/TestResults/** artifacts/runtime-boundaries/** + artifacts/package-capabilities/**/manifest.json + artifacts/package-capabilities/**/consumer-smoke/packages.lock.json + artifacts/package-capabilities/**/packages/* if-no-files-found: error native-abi-x64: diff --git a/eng/prepare-preview.ps1 b/eng/prepare-preview.ps1 index 0d8f2ee..16573fb 100644 --- a/eng/prepare-preview.ps1 +++ b/eng/prepare-preview.ps1 @@ -25,6 +25,11 @@ $output = [System.IO.Path]::GetFullPath($outputPath) $versionOutput = Join-Path $output $Version $packageOutput = Join-Path $versionOutput 'packages' $smokeRoot = Join-Path $versionOutput 'consumer-smoke' +$repositoryCommit = (git -C $root rev-parse HEAD).Trim() +$outputBoundary = [System.IO.Path]::TrimEndingDirectorySeparator($output) + [System.IO.Path]::DirectorySeparatorChar +if (-not ([System.IO.Path]::GetFullPath($versionOutput)).StartsWith($outputBoundary, [StringComparison]::OrdinalIgnoreCase)) { + throw 'The resolved version output must stay inside the explicitly selected output directory.' +} if (Test-Path -LiteralPath $versionOutput) { Remove-Item -LiteralPath $versionOutput -Recurse -Force @@ -46,6 +51,9 @@ foreach ($relativeProject in $projects) { $project, '--configuration', $Configuration, '--no-restore', + '--disable-build-servers', + '-m:1', + '-p:UseSharedCompilation=false', '--output', $packageOutput, '-p:TargetPlatformDisplayName=Windows', "/p:Version=$Version", @@ -101,6 +109,10 @@ $packageEvidence = foreach ($package in $packageFiles | Sort-Object Name) { throw "Package metadata attribution/license is invalid: $($package.Name)" } + if ([string]$metadata.repository.commit -ne $repositoryCommit) { + throw "Package source commit does not match HEAD: $($package.Name)" + } + if ($expectedPackageIds -notcontains [string]$metadata.id) { throw "Unexpected package id: $($metadata.id)" } @@ -130,6 +142,7 @@ $packageEvidence = foreach ($package in $packageFiles | Sort-Object Name) { symbolBytes = $symbol.Length symbolSha256 = (Get-FileHash -LiteralPath $symbol.FullName -Algorithm SHA256).Hash dependencies = $dependencies + repositoryCommit = [string]$metadata.repository.commit entries = @($archive.Entries | ForEach-Object FullName) } } @@ -152,32 +165,165 @@ $consumerNuGetConfig = Join-Path $smokeRoot 'NuGet.config' net10.0-windows enable enable - false + true + false + win-x64;win-arm64 + true - - - - - - - - + + + "@ | Set-Content -LiteralPath $consumerProject -Encoding utf8 -@" +@' +using System.Runtime.Versioning; + using CfSharp; using CfSharp.Storage.Sqlite; -Console.WriteLine(new CloudFilesPlatformInfo(26100, 0, 1536).Supports( - CloudFilesCapability.PlaceholderRangeInfoForHydration)); -Console.WriteLine(new SqliteCloudStateStoreFactory( - Path.Combine(Path.GetTempPath(), "cfsharp-preview.db")).BusyTimeout); -"@ | Set-Content -LiteralPath $consumerProgram -Encoding utf8 +await Smoke.VerifyPagingAsync(); +if (args.Contains("--native", StringComparer.Ordinal)) +{ + if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 16299)) + { + throw new PlatformNotSupportedException("Native package smoke requires Cloud Files."); + } + + await Smoke.VerifyNativeAsync(); +} + +Console.WriteLine("Package capability smoke passed."); + +internal static class Smoke +{ + internal static async Task VerifyPagingAsync() + { + string area = Path.Combine(Path.GetTempPath(), "CfSharp-package-paging", Guid.NewGuid().ToString("N")); + string root = Path.Combine(area, "root"); + Directory.CreateDirectory(root); + try + { + await using ICloudStateStore store = await new SqliteCloudStateStoreFactory(Path.Combine(area, "state.db")) + .OpenAsync(new CloudStateStoreContext(root)); + Guid first = Guid.NewGuid(); + await using (ICloudStateTransaction write = await store.BeginTransactionAsync()) + { + await write.Operations.EnqueueAsync(new(first, CloudStateOperationKind.Create, null, new byte[] { 1 }, DateTimeOffset.UtcNow)); + await write.Operations.EnqueueAsync(new(Guid.NewGuid(), CloudStateOperationKind.Delete, null, new byte[] { 2 }, DateTimeOffset.UtcNow)); + await write.CommitAsync(); + } + + await using ICloudStateTransaction read = await store.BeginTransactionAsync(); + ICloudOperationJournalPaging paging = read.Operations as ICloudOperationJournalPaging + ?? throw new InvalidOperationException("The package lacks official SQLite paging."); + long through = await paging.GetHighWaterSequenceAsync(); + CloudOperationJournalPage page = await paging.ReadPageAsync(0, through, 1); + if (page.Operations.Count != 1 || page.Operations[0].OperationId != first || !page.HasMore || + (await paging.ReadPageAsync(page.LastScannedSequence, through, 1)).Operations.Count != 1 || + (await read.Operations.ListAsync(4)).Count != 2) + { + throw new InvalidOperationException("Package paging lost pending identities or its finite boundary."); + } + } + finally + { + Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); + Directory.Delete(area, recursive: true); + } + } + + [SupportedOSPlatform("windows10.0.16299")] + internal static async Task VerifyNativeAsync() + { + string area = Path.Combine(Path.GetTempPath(), "CfSharp-package-native", Guid.NewGuid().ToString("N")); + string root = Path.Combine(area, "root"); + Directory.CreateDirectory(root); + Guid provider = Guid.NewGuid(); + bool registered = false; + try + { + await using CloudFileSystem system = CloudFileSystem.CreateBuilder(root) + .WithStateStore(new SqliteCloudStateStoreFactory(Path.Combine(area, "state.db"))) + .WithRegistration(SyncRootRegistrationOptions.CreateBuilder("CfSharp Package Smoke", "1.0-test") + .WithProviderId(provider).WithSyncRootIdentity(provider.ToByteArray()) + .WithHydrationPolicy(CloudHydrationPolicy.Progressive).WithPopulationPolicy(CloudPopulationPolicy.Partial) + .WithRootMarkedInSync().Build()).WithContentProvider(new EmptyProvider()).Build(); + await system.StartAsync(); + registered = true; + await system.Root.CreatePlaceholderAsync(CloudDirectoryPlaceholderSpec.CreateBuilder("Docs", "directory") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build()); + await system.GetDirectory("Docs").CreatePlaceholderAsync(CloudFilePlaceholderSpec.CreateBuilder("child.txt", "child", 0).Build()); + CloudLocalChangeFeed feed = system.CreateLocalChangeFeed(); + await feed.StartAsync(); + CloudDirectory original = system.GetDirectory("Docs"); + CloudDirectoryMoveProof proof = await original.PrepareMoveAsync(system.Root, "Moved"); + string applicationProofPath = Path.Combine(area, "move-proof.bin"); + await File.WriteAllBytesAsync(applicationProofPath, proof.Encode()); + Directory.Move(original.FullPath, Path.Combine(root, "Moved")); + CloudDirectoryMoveProof retained = CloudDirectoryMoveProof.Decode(await File.ReadAllBytesAsync(applicationProofPath)); + CloudDirectoryMoveReconciliationResult result = await original.ReconcileMoveAsync(retained); + if (!result.NativeMoveObserved || !result.DurableProjectionCommitted || result.RequiresFullRescan || + result.Outcome is not (CloudDirectoryMoveReconciliationOutcome.Projected or CloudDirectoryMoveReconciliationOutcome.AlreadyProjected)) + { + throw new InvalidOperationException("Package proof recovery failed.", result.Error); + } + + CloudItemMoveResult replay = await original.MoveToAsync(system.Root, "Moved", new CloudMoveOptions(retained)); + if (replay.DirectoryReconciliation?.Outcome != CloudDirectoryMoveReconciliationOutcome.AlreadyProjected) + { + throw new InvalidOperationException("The package cannot replay an original source proof."); + } + + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(15)); + bool observed = false; + while (!observed) + { + CloudLocalChangeScan scan = await feed.BeginScanAsync(timeout.Token); + long after = 0; + do + { + CloudLocalChangePage page = await feed.ReadPageAsync(scan, after, 4, timeout.Token); + if (page.RequiresFullRescan) + { + throw new InvalidOperationException("Package native scan requires unexpected reconciliation."); + } + + observed |= page.Changes.Any(change => change.Kind == CloudLocalChangeKind.Move && change.RelativePath == "Moved"); + after = page.LastScannedSequence; + if (!page.HasMore) + { + break; + } + } while (true); + if (!observed) + { + await Task.Delay(20, timeout.Token); + } + } + } + finally + { + if (registered) + { + CloudSyncRoot.Open(root).Unregister(); + } + Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); + Directory.Delete(area, recursive: true); + } + } + + private sealed class EmptyProvider : ICloudFileContentProvider + { + public ValueTask OpenReadAsync(CloudFileFetchRequest request, CancellationToken cancellationToken) => + ValueTask.FromResult(new MemoryStream()); + } +} +'@ | Set-Content -LiteralPath $consumerProgram -Encoding utf8 @" @@ -205,16 +351,50 @@ $env:USERPROFILE = $consumerUserProfile $env:APPDATA = Join-Path $consumerUserProfile 'AppData/Roaming' $env:DOTNET_CLI_HOME = Join-Path $consumerUserProfile '.dotnet' New-Item -ItemType Directory -Force -Path $env:APPDATA, $env:DOTNET_CLI_HOME | Out-Null +$consumerEvidence = [System.Collections.Generic.List[object]]::new() +$initialCache = Join-Path $versionOutput 'initial-packages' +$lockedCache = Join-Path $versionOutput 'locked-packages' +$hostRuntime = 'win-' + [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant() +if (-not (Get-Command vswhere.exe -ErrorAction SilentlyContinue)) { + $installerDirectory = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio/Installer' + if (Test-Path -LiteralPath (Join-Path $installerDirectory 'vswhere.exe') -PathType Leaf) { + $env:Path = "$installerDirectory;$env:Path" + } +} try { - & dotnet restore $consumerProject --configfile $consumerNuGetConfig ` - -p:TargetPlatformDisplayName=Windows + & dotnet restore $consumerProject --configfile $consumerNuGetConfig --packages $initialCache ` + -p:RestoreLockedMode=false -p:TargetPlatformDisplayName=Windows if ($LASTEXITCODE -ne 0) { - throw "Preview consumer restore failed with exit code $LASTEXITCODE." + throw "Preview consumer initial restore failed with exit code $LASTEXITCODE." } - & dotnet build $consumerProject --configuration $Configuration --no-restore ` + & dotnet restore $consumerProject --configfile $consumerNuGetConfig --packages $lockedCache --locked-mode ` -p:TargetPlatformDisplayName=Windows if ($LASTEXITCODE -ne 0) { - throw "Preview consumer build failed with exit code $LASTEXITCODE." + throw "Preview consumer restore failed with exit code $LASTEXITCODE." + } + foreach ($runtime in @('win-x64', 'win-arm64')) { + foreach ($mode in @('trim', 'aot')) { + $consumerOutput = Join-Path $versionOutput "$mode-$runtime" + & dotnet publish $consumerProject --configuration $Configuration --no-restore --runtime $runtime ` + --self-contained true --disable-build-servers -m:1 -p:UseSharedCompilation=false ` + "-p:PublishAot=$($mode -eq 'aot')" -p:PublishTrimmed=true -p:ILLinkTreatWarningsAsErrors=true ` + -p:TargetPlatformDisplayName=Windows --output $consumerOutput + if ($LASTEXITCODE -ne 0) { + throw "Preview $mode consumer publish failed for $runtime with exit code $LASTEXITCODE." + } + $executable = Join-Path $consumerOutput 'PreviewConsumer.exe' + if (-not (Test-Path -LiteralPath $executable -PathType Leaf)) { + throw "Preview $mode consumer executable is missing for $runtime." + } + $executed = $runtime -eq $hostRuntime + if ($executed) { + & $executable --native + if ($LASTEXITCODE -ne 0) { + throw "Preview $mode native consumer failed for $runtime with exit code $LASTEXITCODE." + } + } + $consumerEvidence.Add([pscustomobject]@{ runtime = $runtime; mode = $mode; compiled = $true; executed = $executed }) + } } } finally { @@ -341,6 +521,8 @@ $manifest = [pscustomobject]@{ releaseNotes = 'release-notes.md' supportMatrix = 'support-matrix.md' consumerSmoke = 'passed' + consumerCapabilities = @($consumerEvidence) + consumerLockSha256 = (Get-FileHash -LiteralPath (Join-Path $smokeRoot 'packages.lock.json') -Algorithm SHA256).Hash msix = [pscustomobject]@{ project = 'samples/CfSharp.SampleProvider.Package/CfSharp.SampleProvider.Package.wapproj' platforms = $platforms -split ';' From 97e2443a699c3353ad187300ba62d12e6405798c Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:33:24 +0800 Subject: [PATCH 19/40] fix(items): preserve operations beyond bounded directory evidence --- docs/placeholders.md | 5 ++ .../Items/CloudDirectory.MovePreparation.cs | 3 +- src/CfSharp/Items/CloudDirectoryMoveProof.cs | 4 +- .../Persistence/CloudDirectoryMoveEvidence.cs | 3 +- .../Persistence/CloudDirectoryProvenance.cs | 54 ++++++++++++++++--- .../CloudDirectoryMoveCompatibilityTests.cs | 39 ++++++++++++++ .../CloudDirectoryMoveProofTests.cs | 13 +++++ 7 files changed, 109 insertions(+), 12 deletions(-) diff --git a/docs/placeholders.md b/docs/placeholders.md index 844252c..c467cb1 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -104,6 +104,11 @@ dumping complete opaque identities or arbitrary user paths into routine diagnost Proof version 1 bounds encoded paths to 32 KiB of UTF-8, identities to the native 4 KiB limit, and known-member manifests to 64 MiB. Unknown protocol versions fail closed. Preparation is metadata-only and does not freeze child content, identity writes, or later namespace moves. +These evidence bounds do not reduce existing directory conversion or ordinary `MoveToAsync` +availability. If a valid native path or known subtree exceeds the recovery format's capacity, +normal source-present operations retain their existing behavior and omit unusable live provenance; +explicit preparation reports `NotSupportedException`. Immutable earlier proofs, receipts, official +item rows, and pending operations remain intact. Placeholder operations use immutable, kind-specific specifications rather than exposing native unions and flag combinations directly to application code. diff --git a/src/CfSharp/Items/CloudDirectory.MovePreparation.cs b/src/CfSharp/Items/CloudDirectory.MovePreparation.cs index dd45016..8338cb5 100644 --- a/src/CfSharp/Items/CloudDirectory.MovePreparation.cs +++ b/src/CfSharp/Items/CloudDirectory.MovePreparation.cs @@ -20,7 +20,8 @@ public sealed partial class CloudDirectory /// The destination belongs to another file system or the name is invalid. /// The root, destination, or durable/native identity is unsuitable. /// - /// Storage cannot supply the complete native binding, known subtree metadata exceeds 64 MiB, + /// Storage cannot supply the complete native binding, an encoded evidence path exceeds 32 KiB, + /// known subtree metadata exceeds 64 MiB, /// or durable recovery records use an unsupported protocol version. /// /// Native metadata, path protection, or durable preparation failed. diff --git a/src/CfSharp/Items/CloudDirectoryMoveProof.cs b/src/CfSharp/Items/CloudDirectoryMoveProof.cs index 22c3cde..d26005a 100644 --- a/src/CfSharp/Items/CloudDirectoryMoveProof.cs +++ b/src/CfSharp/Items/CloudDirectoryMoveProof.cs @@ -156,7 +156,7 @@ internal static string CanonicalPath(string path) string canonical = CloudRemotePathValidation.Canonicalize(path, nameof(path)).Replace('/', '\\'); if (Utf8.GetByteCount(canonical) > 32768) { - throw new ArgumentException("The directory evidence path is too long.", nameof(path)); + throw new CloudDirectoryEvidenceUnavailableException("The directory evidence path exceeds the supported UTF-8 metadata limit."); } return canonical; @@ -167,7 +167,7 @@ internal static void WriteText(BinaryWriter writer, string value) byte[] bytes = Utf8.GetBytes(value); if (bytes.Length > 32768) { - throw new InvalidDataException("The directory evidence path is too long."); + throw new CloudDirectoryEvidenceUnavailableException("The directory evidence path exceeds the supported UTF-8 metadata limit."); } writer.Write(bytes.Length); diff --git a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs index 7e9ae21..5c117b1 100644 --- a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs +++ b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs @@ -15,7 +15,8 @@ internal static class CloudDirectoryMoveEvidence internal static string ReceiptName(Guid id) => $"cfsharp/namespace/preparations/{id:N}/receipt"; internal static string IntentName(string source, string target) => "cfsharp/namespace/intents/" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes( - CloudDirectoryEvidenceCodec.CanonicalPath(source).ToUpperInvariant() + "\0" + CloudDirectoryEvidenceCodec.CanonicalPath(target)))); + CloudRemotePathValidation.Canonicalize(source, nameof(source)).Replace('/', '\\').ToUpperInvariant() + "\0" + + CloudRemotePathValidation.Canonicalize(target, nameof(target)).Replace('/', '\\')))); internal static async ValueTask ReadIntentAsync(ICloudStateStore store, string source, string target, CancellationToken cancellationToken) diff --git a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs index 2519270..c8a6bb1 100644 --- a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs +++ b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs @@ -102,9 +102,7 @@ internal static async ValueTask RetainProjectionAsync(ICloudStateTransaction tra Guid scope = await CloudDirectoryMoveEvidence.GetScopeAsync(transaction, true, cancellationToken).ConfigureAwait(false); CloudDirectoryProvenance provenance = new(Guid.NewGuid(), scope, item.ItemId, Path.GetRelativePath(syncRootPath, captured.ActualPath), captured.Binding, captured.Identity); - await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(BindingName(item.ItemId), provenance.Encode(), DateTimeOffset.UtcNow), - cancellationToken).ConfigureAwait(false); - await RefreshMembersAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); + await RetainMetadataAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); } // Refresh known membership of already captured ancestors using official transactional @@ -164,18 +162,58 @@ internal static async ValueTask ProjectPathsAsync(ICloudStateTransaction transac provenance = provenance with { EvidenceId = Guid.NewGuid(), RelativePath = targetPath }; } - await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(BindingName(item.ItemId), provenance.Encode(), DateTimeOffset.UtcNow), - cancellationToken).ConfigureAwait(false); - await RefreshMembersAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); + await RetainMetadataAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); } } + private static async ValueTask RetainMetadataAsync(ICloudStateTransaction transaction, CloudDirectoryProvenance provenance, + CancellationToken cancellationToken) + { + byte[] binding; + byte[] manifest; + try + { + // Compute both bounded records before replacing either existing record. Recovery + // metadata limits must not fail an otherwise valid legacy namespace operation. + binding = provenance.Encode(); + IReadOnlyList members = await transaction.Items.ListSubtreeAsync(provenance.RelativePath, cancellationToken).ConfigureAwait(false); + manifest = CloudDirectoryMoveEvidence.EncodeMembers(provenance.EvidenceId, provenance.RootItemId, provenance.RelativePath, members); + } + catch (CloudDirectoryEvidenceUnavailableException) + { + await RemoveUnusableMetadataAsync(transaction, provenance.RootItemId, cancellationToken).ConfigureAwait(false); + return; + } + + DateTimeOffset now = DateTimeOffset.UtcNow; + await transaction.Checkpoints.UpsertAsync(new(BindingName(provenance.RootItemId), binding, now), cancellationToken).ConfigureAwait(false); + await transaction.Checkpoints.UpsertAsync(new(MembersName(provenance.RootItemId), manifest, now), cancellationToken).ConfigureAwait(false); + } + private static async ValueTask RefreshMembersAsync(ICloudStateTransaction transaction, CloudDirectoryProvenance provenance, CancellationToken cancellationToken) { IReadOnlyList members = await transaction.Items.ListSubtreeAsync(provenance.RelativePath, cancellationToken).ConfigureAwait(false); - await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(MembersName(provenance.RootItemId), - CloudDirectoryMoveEvidence.EncodeMembers(provenance.EvidenceId, provenance.RootItemId, provenance.RelativePath, members), DateTimeOffset.UtcNow), + byte[] manifest; + try + { + manifest = CloudDirectoryMoveEvidence.EncodeMembers(provenance.EvidenceId, provenance.RootItemId, provenance.RelativePath, members); + } + catch (CloudDirectoryEvidenceUnavailableException) + { + await RemoveUnusableMetadataAsync(transaction, provenance.RootItemId, cancellationToken).ConfigureAwait(false); + return; + } + + await transaction.Checkpoints.UpsertAsync(new CloudStateCheckpoint(MembersName(provenance.RootItemId), manifest, DateTimeOffset.UtcNow), cancellationToken).ConfigureAwait(false); } + + private static async ValueTask RemoveUnusableMetadataAsync(ICloudStateTransaction transaction, Guid itemId, CancellationToken cancellationToken) + { + // Remove only mutable live provenance. Original immutable preparations and receipts, + // official item rows, and pending journal entries remain in their caller's transaction. + await transaction.Checkpoints.RemoveAsync(BindingName(itemId), cancellationToken).ConfigureAwait(false); + await transaction.Checkpoints.RemoveAsync(MembersName(itemId), cancellationToken).ConfigureAwait(false); + } } diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCompatibilityTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCompatibilityTests.cs index d614368..841d2ed 100644 --- a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCompatibilityTests.cs +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCompatibilityTests.cs @@ -7,6 +7,45 @@ namespace CfSharp.IntegrationTests; [SupportedOSPlatform("windows10.0.16299")] public sealed partial class CloudDirectoryMoveTests { + [Fact] + public async Task BoundedUtf8HistoryPreservesLegacyManagedDirectoryConversionAndMove() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectoryMoveProof original = await root.FileSystem.GetDirectory("Docs").PrepareMoveAsync(root.FileSystem.Root, "Other"); + CloudDirectory source = root.FileSystem.GetDirectory("Ordinary"); + string suffix = string.Join('\\', Enumerable.Repeat(new string('æ–‡', 250), 45)); + Guid deletedId = Guid.NewGuid(); + DateTimeOffset deletedAt = DateTimeOffset.UtcNow; + await using (ICloudStateTransaction seed = await root.Store.BeginTransactionAsync()) + { + // Official historical rows can retain paths outside the bounded recovery format, + // independently of the current machine's native long-path support. + await seed.Items.UpsertAsync(new(deletedId, "deleted", Path.Combine("Ordinary", suffix), + CloudItemKind.File, "deleted-revision", 42, true, deletedAt)); + await seed.CommitAsync(); + } + + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Create("ordinary-directory"); + await source.ConvertToPlaceholderAsync(identity); + await Assert.ThrowsAsync(() => source.PrepareMoveAsync(root.FileSystem.Root, "Moved").AsTask()); + CloudItemMoveResult moved = await source.MoveToAsync(root.FileSystem.Root, "Moved"); + Assert.Null(moved.DirectoryReconciliation); + Assert.Equal(identity.ItemId, moved.Snapshot.ItemId); + Assert.False(Directory.Exists(source.FullPath)); + Assert.True(Directory.Exists(Path.Combine(root.RootPath, "Moved"))); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal("Moved", (await verify.Items.GetByItemIdAsync(identity.ItemId))!.RelativePath); + CloudItemState deleted = (await verify.Items.GetByItemIdAsync(deletedId))!; + Assert.Equal(Path.Combine("Moved", suffix), deleted.RelativePath); + Assert.True(deleted.IsTombstone); + Assert.Equal("deleted-revision", deleted.RemoteRevision); + Assert.Equal(42, deleted.LocalFileId); + Assert.True(deleted.UpdatedAt >= deletedAt); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(identity.ItemId))); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.MembersName(identity.ItemId))); + Assert.Equal(original.Encode(), (await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ProofName(original.ProofId)))!.Value.ToArray()); + } + [Fact] public async Task PublishedPreviewThreeStateSurvivesDirectoryFeedProjectionProofReplayAndRuntimeRestart() { diff --git a/tests/CfSharp.Tests/Persistence/CloudDirectoryMoveProofTests.cs b/tests/CfSharp.Tests/Persistence/CloudDirectoryMoveProofTests.cs index c657035..e43c596 100644 --- a/tests/CfSharp.Tests/Persistence/CloudDirectoryMoveProofTests.cs +++ b/tests/CfSharp.Tests/Persistence/CloudDirectoryMoveProofTests.cs @@ -4,6 +4,19 @@ namespace CfSharp.Tests.Persistence; public sealed class CloudDirectoryMoveProofTests { + [Fact] + public void EvidenceCapacityDoesNotRestrictCanonicalIntentIndexOrRelaxBadPathValidation() + { + string path = string.Join('\\', Enumerable.Repeat(new string('æ–‡', 250), 45)); + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Create("remote"); + Assert.Throws(() => new CloudDirectoryMoveProof(Guid.NewGuid(), Guid.NewGuid(), + path, "Target", identity.ItemId, new CloudLocalFileBinding(1, Guid.NewGuid(), Guid.NewGuid()), identity.Encode())); + string index = CloudDirectoryMoveEvidence.IntentName(path, "Target"); + Assert.Equal("cfsharp/namespace/intents/".Length + 64, index.Length); + Assert.Equal(index, CloudDirectoryMoveEvidence.IntentName(path.Replace('\\', '/'), "Target")); + Assert.Throws(() => CloudDirectoryMoveEvidence.IntentName("..\\Docs", "Target")); + } + [Fact] public void SerializationOwnsCompleteBindingAndIdentityAndRejectsMalformedLengths() { From 78da2e8ddc56b62b2a42c4134e8416f574a51f60 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 07:45:30 +0800 Subject: [PATCH 20/40] test(sqlite): drain and reap abrupt crash subprocesses --- .../SqliteCloudStateStoreTests.cs | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs index 3565844..06e0c8b 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/SqliteCloudStateStoreTests.cs @@ -825,11 +825,28 @@ private static async Task ExecuteSqlAsync(string databasePath, string sql) using Process process = new() { StartInfo = startInfo }; Assert.True(process.Start()); + // FailFast diagnostics can fill a redirected Windows pipe before the child terminates. + // Drain both streams concurrently with the exit wait, and always reap a timed-out child + // before fixture disposal tries to remove its still-open SQLite files. + Task standardOutput = process.StandardOutput.ReadToEndAsync(); + Task standardError = process.StandardError.ReadToEndAsync(); using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(30)); - await process.WaitForExitAsync(timeout.Token); - string output = await process.StandardOutput.ReadToEndAsync() + - await process.StandardError.ReadToEndAsync(); - return (process.ExitCode, output); + try + { + await process.WaitForExitAsync(timeout.Token); + string output = await standardOutput + await standardError; + return (process.ExitCode, output); + } + finally + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + await process.WaitForExitAsync(); + } + + await Task.WhenAll(standardOutput, standardError); + } } private static async Task ExecuteScalarInt64Async(string databasePath, string sql) From 74e2d9bf0db6f7521917fa7de4525b92b1c2f2f9 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:09:15 +0800 Subject: [PATCH 21/40] test(local-changes): verify scans recover from SQLite query faults --- .../CloudLocalChangeScanTests.cs | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanTests.cs index 056784c..c41fe97 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanTests.cs @@ -1,3 +1,5 @@ +using Microsoft.Data.Sqlite; + namespace CfSharp.Storage.Sqlite.Tests; public sealed partial class CloudLocalChangeFeedTests @@ -147,6 +149,64 @@ await write.Operations.EnqueueAsync(new CloudOperationJournalEntry(operationId, await verify.RollbackAsync(); } + [Fact] + public async Task SqliteQueryFailureReleasesScanTransactionsAndPreservesOriginalJournal() + { + await using ICloudStateStore store = await OpenStoreAsync(); + CloudOperationJournalEntry[] expected = await SeedChangesAsync(store, 2); + await using CloudLocalChangeFeed feed = CreateFeed(store, new FakeSource()); + await feed.StartAsync(); + CloudLocalChangeScan scan = await feed.BeginScanAsync(); + await using SqliteConnection repair = new(new SqliteConnectionStringBuilder + { + DataSource = _databasePath, + Pooling = false, + }.ToString()); + await repair.OpenAsync(); + await SqliteSchema.ConfigureConnectionAsync(repair, 5000, CancellationToken.None); + await using SqliteCommand command = repair.CreateCommand(); + // Interrupt the real SQLite query without deleting or rewriting any journal row. + // Restoring the table and reusing this scan must remain possible after both failures. + command.CommandText = "ALTER TABLE operations RENAME TO interrupted_operations;"; + await command.ExecuteNonQueryAsync(); + try + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(30)); + SqliteCloudStateStoreException beginFailure = await Assert.ThrowsAsync( + async () => await feed.BeginScanAsync(timeout.Token)); + SqliteCloudStateStoreException pageFailure = await Assert.ThrowsAsync( + async () => await feed.ReadPageAsync(scan, 0, 1, timeout.Token)); + Assert.Equal(1, beginFailure.SqliteErrorCode); + Assert.Equal(1, pageFailure.SqliteErrorCode); + Assert.IsType(beginFailure.InnerException); + Assert.IsType(pageFailure.InnerException); + } + finally + { + command.CommandText = "ALTER TABLE interrupted_operations RENAME TO operations;"; + await command.ExecuteNonQueryAsync(); + } + + using CancellationTokenSource recoveryTimeout = new(TimeSpan.FromSeconds(30)); + CloudLocalChangePage page = await feed.ReadPageAsync(scan, 0, 2, recoveryTimeout.Token); + Assert.False(page.RequiresFullRescan); + Assert.False(page.HasMore); + Assert.Equal(expected.Select(row => row.OperationId), page.Changes.Select(change => change.OperationId)); + await using ICloudStateTransaction verify = await store.BeginTransactionAsync(recoveryTimeout.Token); + IReadOnlyList retained = await verify.Operations.ListAsync(2, recoveryTimeout.Token); + Assert.Equal(expected.Select(row => row.OperationId), retained.Select(row => row.OperationId)); + for (int index = 0; index < expected.Length; index++) + { + Assert.Equal(expected[index].Sequence, retained[index].Sequence); + Assert.Equal(expected[index].Payload.ToArray(), retained[index].Payload.ToArray()); + Assert.Equal(expected[index].CreatedAt, retained[index].CreatedAt); + Assert.Equal(expected[index].AttemptCount, retained[index].AttemptCount); + Assert.Equal(expected[index].RetryAfter, retained[index].RetryAfter); + } + + await verify.RollbackAsync(recoveryTimeout.Token); + } + private static async Task SeedChangesAsync(ICloudStateStore store, int count) { await using ICloudStateTransaction transaction = await store.BeginTransactionAsync(); From 20a8b96507b19bb8734036e7ce5a97f3e1ac7b34 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:27:55 +0800 Subject: [PATCH 22/40] test(local-changes): await durable echo processing deterministically --- .../CloudLocalChangeFeedTests.cs | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs index f4cc17d..763c663 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs @@ -142,35 +142,42 @@ await feed.SuppressProviderEchoAsync( expectedObservationCount: 2); await source.EmitAsync(new(LocalChangeSourceAction.Modified, "replayed.txt")); await source.EmitAsync(new(LocalChangeSourceAction.Modified, "replayed.txt")); - await WaitForJournalStateAsync(store, expectedOperations: 0, expectedSuppressions: 0); + // Suppressed observations produce no journal row to await. The public drain boundary + // establishes completion without racing a polling deadline against worker scheduling. + await feed.DisposeAsync(); await using (ICloudStateTransaction transaction = await store.BeginTransactionAsync()) { Assert.Empty(await transaction.Operations.ListAsync(10)); Assert.Empty(await transaction.EchoSuppressions.ListActiveAsync(DateTimeOffset.UtcNow)); + LocalChangeCheckpoint checkpoint = LocalChangeCheckpoint.Decode( + (await transaction.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value); + Assert.Equal(2, checkpoint.Observation); + Assert.False(checkpoint.RequiresFullRescan); await transaction.RollbackAsync(); } - await source.EmitAsync(new(LocalChangeSourceAction.Modified, "replayed.txt")); - await WaitForJournalStateAsync(store, expectedOperations: 1, expectedSuppressions: 0); - await using (ICloudStateTransaction afterBudget = await store.BeginTransactionAsync()) - { - Assert.Single(await afterBudget.Operations.ListAsync(10)); - await afterBudget.RollbackAsync(); - } - - await feed.AcknowledgeAsync( - (await feed.ReadBatchAsync()).Changes.Select(change => change.OperationId)); - await feed.SuppressProviderEchoAsync( + FakeSource restartedSource = new(); + await using CloudLocalChangeFeed restarted = CreateFeed(store, restartedSource); + await restarted.StartAsync(); + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(30)); + await restartedSource.EmitAsync(new(LocalChangeSourceAction.Modified, "replayed.txt")); + CloudLocalChangeBatch afterBudget = await restarted.ReadBatchAsync(timeout.Token); + Assert.Equal(CloudLocalChangeKind.ContentUpdate, Assert.Single(afterBudget.Changes).Kind); + Assert.False(afterBudget.RequiresFullRescan); + + await restarted.AcknowledgeAsync(afterBudget.Changes.Select(change => change.OperationId), timeout.Token); + await restarted.SuppressProviderEchoAsync( CloudStateOperationKind.ContentUpdate, "kind-sensitive.txt", DateTimeOffset.UtcNow.AddMinutes(1)); - await source.EmitAsync(new(LocalChangeSourceAction.Created, "kind-sensitive.txt")); - await WaitForJournalStateAsync(store, expectedOperations: 1, expectedSuppressions: 1); + await restartedSource.EmitAsync(new(LocalChangeSourceAction.Created, "kind-sensitive.txt")); + Assert.Equal(CloudLocalChangeKind.Create, Assert.Single((await restarted.ReadBatchAsync(timeout.Token)).Changes).Kind); await using ICloudStateTransaction kindMismatch = await store.BeginTransactionAsync(); CloudOperationJournalEntry kindSensitiveOperation = Assert.Single( await kindMismatch.Operations.ListAsync(10)); Assert.Equal(CloudStateOperationKind.Create, kindSensitiveOperation.Kind); + Assert.Equal(1, Assert.Single(await kindMismatch.EchoSuppressions.ListActiveAsync(DateTimeOffset.UtcNow)).RemainingObservations); await kindMismatch.RollbackAsync(); } From 4b08f327a61b3b4f1e94a5a70911d5a54b7a3302 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:38:44 +0800 Subject: [PATCH 23/40] fix(lifecycle): drain owned rename notifications before shutdown --- docs/local-change-feed.md | 7 + docs/sync-root-lifecycle.md | 7 +- src/CfSharp/FileSystem/CloudFileSystem.cs | 103 ++++++------ .../LocalChanges/CloudLocalChangeFeed.cs | 43 ++++- .../LocalChanges/CloudLocalChangeModels.cs | 6 +- .../CloudLocalChangeOwnerShutdownTests.cs | 153 ++++++++++++++++++ 6 files changed, 264 insertions(+), 55 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudLocalChangeOwnerShutdownTests.cs diff --git a/docs/local-change-feed.md b/docs/local-change-feed.md index 5242ccd..8db0ac9 100644 --- a/docs/local-change-feed.md +++ b/docs/local-change-feed.md @@ -3,6 +3,13 @@ The local change feed turns Windows directory notifications into an explicit, bounded application queue. It is a coordination aid, not a lossless replacement for reconciliation. +Disposal stops notification production and drains accepted events while the state store remains +open, including queue-tail renames during owner shutdown. If draining times out or fails, the feed +persists a full-rescan marker before canceling its worker. Persisting that fence can wait for an +active store transaction beyond the worker's `ShutdownTimeout`. If fence persistence fails, disposal +reports the failure, leaves the worker uncanceled, and retains the owner's store for a disposal retry; +the retry must complete the drain and persist the fence before the store is closed. + ## Start one feed per sync root ```csharp diff --git a/docs/sync-root-lifecycle.md b/docs/sync-root-lifecycle.md index eda732f..49a20de 100644 --- a/docs/sync-root-lifecycle.md +++ b/docs/sync-root-lifecycle.md @@ -28,8 +28,11 @@ registration after the call succeeds. registration, and connects the optional content provider. Work admitted after start holds an explicit operation lease. -Disposal rejects new work, waits for admitted operations, stops the provider session, and closes -durable state. It intentionally leaves the sync-root registration installed. +Disposal rejects new public work, stops the local notification producer, and drains accepted +notifications and admitted operations before stopping the provider session and closing durable +state. Accepted renames retain internal path-scope admission during `Stopping`; disposal does not +hold the lifecycle admission lock while awaiting their completion. Concurrent disposal attempts +remain serialized. It intentionally leaves the sync-root registration installed. ## Unregister only for removal diff --git a/src/CfSharp/FileSystem/CloudFileSystem.cs b/src/CfSharp/FileSystem/CloudFileSystem.cs index dffc2ae..df77f8c 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.cs @@ -39,6 +39,7 @@ namespace CfSharp; public sealed partial class CloudFileSystem : IDisposable, IAsyncDisposable { private readonly SemaphoreSlim _lifecycleGate = new(1, 1); + private readonly SemaphoreSlim _disposalGate = new(1, 1); private readonly CloudItemOperationCoordinator _operationCoordinator = new(); private readonly AsyncLocal _operationContext = new(); private readonly ICloudStateStoreFactory _stateStoreFactory; @@ -116,7 +117,8 @@ public CloudLocalChangeFeed CreateLocalChangeFeed( } } }, - async (scopes, cancellationToken) => await AcquireOperationAsync(scopes, cancellationToken).ConfigureAwait(false)); + async (scopes, cancellationToken) => await AcquireOperationCoreAsync( + scopes, allowLocalChangeDrain: true, cancellationToken).ConfigureAwait(false)); } } @@ -311,7 +313,8 @@ public async ValueTask StartAsync(CancellationToken cancellationToken = default) /// context should call instead and await it. /// /// - /// More than one owned resource failed during disposal. Every resource is still attempted. + /// More than one owned resource failed during disposal. Resources are attempted in dependency + /// order; an unfinished or failed feed drain keeps its dependent resources alive for a retry. /// Failed resources remain owned so disposal can be retried. /// public void Dispose() @@ -325,48 +328,53 @@ public void Dispose() /// An operation that completes after admitted item operations drain and the provider session /// and state store terminate. /// + /// + /// Public operation admission stops immediately. Accepted local notifications retain internal + /// path-scope admission until their feed drains, without holding the lifecycle admission lock. + /// A failed feed drain retains dependent resources for a later disposal attempt. + /// /// - /// More than one owned resource failed during disposal. Every resource is still attempted. + /// More than one owned resource failed during disposal. Resources are attempted in dependency order. /// Failed resources remain owned so disposal can be retried. /// public async ValueTask DisposeAsync() { - await _lifecycleGate.WaitAsync().ConfigureAwait(false); + // Serialize disposal attempts independently of admission. The feed must acquire the + // lifecycle gate while draining accepted renames, including after public admission stops. + await _disposalGate.WaitAsync().ConfigureAwait(false); try { - if (LifecycleState is CloudFileSystemLifecycleState.Disposed) + await _lifecycleGate.WaitAsync().ConfigureAwait(false); + try { - return; - } - - Volatile.Write(ref _state, (int)CloudFileSystemLifecycleState.Stopping); - _contentConfirmationStopping.Cancel(); - } - finally - { - _lifecycleGate.Release(); - } - - Task operationsDrained = Volatile.Read(ref _activeOperations) == 0 - ? Task.CompletedTask - : Volatile.Read(ref _operationsDrained)?.Task ?? Task.CompletedTask; - await operationsDrained.ConfigureAwait(false); + if (LifecycleState is CloudFileSystemLifecycleState.Disposed) + { + return; + } - await _lifecycleGate.WaitAsync().ConfigureAwait(false); - try - { - if (LifecycleState is CloudFileSystemLifecycleState.Disposed) + Volatile.Write(ref _state, (int)CloudFileSystemLifecycleState.Stopping); + _contentConfirmationStopping.Cancel(); + } + finally { - return; + _lifecycleGate.Release(); } IReadOnlyList failures = await DisposeOwnedResourcesAsync() .ConfigureAwait(false); - if (failures.Count == 0) + await _lifecycleGate.WaitAsync().ConfigureAwait(false); + try + { + if (failures.Count == 0) + { + _contentConfirmationStopping.Dispose(); + Volatile.Write(ref _state, (int)CloudFileSystemLifecycleState.Disposed); + GC.SuppressFinalize(this); + } + } + finally { - _contentConfirmationStopping.Dispose(); - Volatile.Write(ref _state, (int)CloudFileSystemLifecycleState.Disposed); - GC.SuppressFinalize(this); + _lifecycleGate.Release(); } if (failures.Count == 1) @@ -383,7 +391,7 @@ public async ValueTask DisposeAsync() } finally { - _lifecycleGate.Release(); + _disposalGate.Release(); } } @@ -408,6 +416,7 @@ await AcquireOperationCoreAsync( private async ValueTask AcquireOperationCoreAsync( IEnumerable scopes, + bool allowLocalChangeDrain = false, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(scopes); @@ -416,7 +425,13 @@ private async ValueTask AcquireOperationCoreAsync await _lifecycleGate.WaitAsync(cancellationToken).ConfigureAwait(false); try { - EnsureStarted(); + // Only the owned feed receives this internal admission path. Public operations + // remain rejected in Stopping, while accepted notifications retain path exclusion + // and a store-lifetime lease until the feed has completely drained. + if (!allowLocalChangeDrain || LifecycleState != CloudFileSystemLifecycleState.Stopping || _stateStore is null) + { + EnsureStarted(); + } CloudFileSystemOperationLease? currentContext = _operationContext.Value; if (currentContext is not null && currentContext.Owner is not null && @@ -572,36 +587,30 @@ private async ValueTask> DisposeOwnedResourcesAsync() if (localChangeFeed is not null) { - bool disposed = false; try { await localChangeFeed.DisposeAsync().ConfigureAwait(false); - disposed = true; } catch (Exception exception) { (failures ??= []).Add(exception); - if (!localChangeFeed.DisposeCompletion.IsCompleted) - { - // The feed may still be draining a journal transaction. Its owner must keep - // the state store alive until the deferred completion callback runs. - lock (_localChangeFeedGate) - { - _localChangeFeed ??= localChangeFeed; - } - return failures; - } - } - - if (!disposed) - { lock (_localChangeFeedGate) { _localChangeFeed ??= localChangeFeed; } + // Keep the store available for both deferred worker completion and a durable + // loss-marker retry, even if the worker completed concurrently with the failure. + return failures; } } + // The stopped producer cannot admit more feed operations. Drain remaining facade + // leases before disconnecting callbacks or disposing the store they still reference. + Task operationsDrained = Volatile.Read(ref _activeOperations) == 0 + ? Task.CompletedTask + : Volatile.Read(ref _operationsDrained)?.Task ?? Task.CompletedTask; + await operationsDrained.ConfigureAwait(false); + if (_runtimeSession is not null) { bool disposed = false; diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs index e10a83a..4e06dd5 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs @@ -378,6 +378,12 @@ public void Dispose() } /// Stops the watcher and releases all feed-owned resources. + /// + /// Accepted notifications drain before worker cancellation. A failed or timed-out drain persists + /// a full-rescan fence before cancellation; this store transaction may outlast the worker timeout. + /// If fence persistence fails, the worker remains uncanceled and disposal must be retried while + /// the state store remains open. Concurrent disposal callers wait for worker cleanup. + /// /// /// The processor did not stop within the configured shutdown timeout. Deferred cleanup keeps /// feed-owned resources alive until the processor exits. @@ -387,10 +393,16 @@ public async ValueTask DisposeAsync() if (Interlocked.Exchange(ref _disposed, 1) != 0) { await _disposeCompletion.Task.ConfigureAwait(false); + if (HasUnpersistedLoss) + { + await MarkRescanRequiredAsync(CancellationToken.None).ConfigureAwait(false); + _onDisposed?.Invoke(this); + } return; } Exception? failure = null; + bool cancelProcessor = true; try { await _source.DisposeAsync().ConfigureAwait(false); @@ -408,17 +420,35 @@ public async ValueTask DisposeAsync() { await _processorTask.WaitAsync(_options.ShutdownTimeout).ConfigureAwait(false); } - } - catch (OperationCanceledException) - { + + if (HasUnpersistedLoss) + { + await MarkRescanRequiredAsync(CancellationToken.None).ConfigureAwait(false); + } } catch (Exception exception) { failure = exception; + // Cancellation can abandon a notification before it creates a journal row. Fence + // durable consumers first, while the owner still keeps the store alive. If persistence + // fails, leave the worker uncanceled and retain ownership for a later marker retry. + cancelProcessor = false; + try + { + await MarkRescanRequiredAsync(CancellationToken.None).ConfigureAwait(false); + cancelProcessor = true; + } + catch (Exception markerFailure) + { + failure = new AggregateException("The local-change feed could not persist shutdown loss.", exception, markerFailure); + } } finally { - _shutdown.Cancel(); + if (cancelProcessor) + { + _shutdown.Cancel(); + } _sourceEvents.Writer.TryComplete(); if (_processorTask is null || _processorTask.IsCompleted) { @@ -457,7 +487,10 @@ private void CompleteDispose() { _availability.Writer.TryComplete(); _shutdown.Dispose(); - _onDisposed?.Invoke(this); + if (!HasUnpersistedLoss) + { + _onDisposed?.Invoke(this); + } _disposeCompletion.TrySetResult(null); } diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeModels.cs b/src/CfSharp/LocalChanges/CloudLocalChangeModels.cs index 9a45690..b2a3712 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeModels.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeModels.cs @@ -34,7 +34,11 @@ public sealed record CloudLocalChangeFeedOptions /// Gets the maximum number of journal entries returned by one read. public int BatchSize { get; init; } = 64; - /// Gets the maximum time allowed for cooperative watcher shutdown. + /// Gets the maximum time allowed for the notification worker to drain during shutdown. + /// + /// A timed-out drain persists a full-rescan fence before canceling the worker. Waiting for that + /// durable store transaction and deferred cleanup can extend disposal beyond this timeout. + /// public TimeSpan ShutdownTimeout { get; init; } = TimeSpan.FromSeconds(30); internal void Validate() diff --git a/tests/CfSharp.IntegrationTests/CloudLocalChangeOwnerShutdownTests.cs b/tests/CfSharp.IntegrationTests/CloudLocalChangeOwnerShutdownTests.cs new file mode 100644 index 0000000..aa01380 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudLocalChangeOwnerShutdownTests.cs @@ -0,0 +1,153 @@ +using System.Reflection; +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed class CloudLocalChangeOwnerShutdownTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OwnerDrainsTailRenameWhileStoppingAndRejectsNewPublicWork(bool directory) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + string before = directory ? "Docs" : "Docs\\child.txt"; + string after = directory ? "Moved" : "Docs\\renamed.txt"; + TailRenameSource source = new(before, after); + CloudLocalChangeFeed feed = InstallSource(root, source); + await feed.StartAsync(); + if (directory) + { + Directory.Move(Path.Combine(root.RootPath, before), Path.Combine(root.RootPath, after)); + } + else + { + File.Move(Path.Combine(root.RootPath, before), Path.Combine(root.RootPath, after)); + } + + Task shutdown = root.FileSystem.DisposeAsync().AsTask(); + await source.Stopping.Task.WaitAsync(TimeSpan.FromSeconds(10)); + Task concurrentShutdown = root.FileSystem.DisposeAsync().AsTask(); + Assert.Equal(CloudFileSystemLifecycleState.Stopping, root.FileSystem.LifecycleState); + Assert.Throws(() => root.FileSystem.GetDirectory("Docs")); + source.Release.TrySetResult(); + try + { + await Task.WhenAll(shutdown, concurrentShutdown).WaitAsync(TimeSpan.FromSeconds(10)); + } + finally + { + await feed.DisposeCompletion.WaitAsync(TimeSpan.FromSeconds(10)); + } + + await root.RestartAsync(); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + CloudOperationJournalEntry move = Assert.Single(await verify.Operations.ListAsync(100)); + Assert.Equal(CloudStateOperationKind.Move, move.Kind); + LocalChangePayload payload = LocalChangePayload.Decode(move.Payload); + Assert.Equal(before, payload.PreviousRelativePath); + Assert.Equal(after, payload.RelativePath); + Assert.Equal(directory, payload.IsDirectory); + Assert.False(LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + if (directory) + { + Assert.Equal("Moved", (await verify.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!.RelativePath); + Assert.NotNull(await verify.Items.GetByRelativePathAsync("Moved\\child.txt")); + } + } + + private static CloudLocalChangeFeed InstallSource(DirectoryMoveTestRoot root, TailRenameSource source) + { + CloudLocalChangeFeed feed = root.FileSystem.CreateLocalChangeFeed(new() { ShutdownTimeout = TimeSpan.FromSeconds(1) }); + // Replace only the native producer. The owned feed retains the production operation + // acquisition delegate and disposal callback, so the test exercises the owner lifecycle. + typeof(CloudLocalChangeFeed).GetField("_source", BindingFlags.NonPublic | BindingFlags.Instance)!.SetValue(feed, source); + return feed; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task FailedDrainPersistsLossBeforeCancellationAndRetainsStoreForRetry(bool markerCommitFails) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + TailRenameSource source = new("Docs", "Moved"); + CloudLocalChangeFeed feed = InstallSource(root, source); + await feed.StartAsync(); + using CloudFileSystem.CloudFileSystemOperationLease blocker = await root.FileSystem.AcquireOperationAsync( + [CloudItemOperationScope.Subtree(Path.Combine(root.RootPath, "Docs"))]); + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + bool fenceCommittedBeforeCancellation = false; + root.Faults.BeforeCommit = () => + { + if (markerCommitFails) + { + throw new IOException("Injected shutdown fence commit failure."); + } + }; + root.Faults.AfterCommit = () => fenceCommittedBeforeCancellation = !source.Cancellation.IsCancellationRequested; + try + { + Task shutdown = root.FileSystem.DisposeAsync().AsTask(); + await source.Stopping.Task.WaitAsync(TimeSpan.FromSeconds(10)); + source.Release.TrySetResult(); + if (markerCommitFails) + { + AggregateException failure = await Assert.ThrowsAsync(() => shutdown.WaitAsync(TimeSpan.FromSeconds(10))); + Assert.IsType(failure.InnerExceptions[0]); + Assert.IsType(failure.InnerExceptions[1]); + Assert.False(source.Cancellation.IsCancellationRequested); + Assert.False(feed.DisposeCompletion.IsCompleted); + } + else + { + await Assert.ThrowsAsync(() => shutdown.WaitAsync(TimeSpan.FromSeconds(10))); + Assert.True(fenceCommittedBeforeCancellation); + Assert.True(source.Cancellation.IsCancellationRequested); + await feed.DisposeCompletion.WaitAsync(TimeSpan.FromSeconds(10)); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Empty(await verify.Operations.ListAsync(100)); + Assert.True(LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + } + + Assert.Equal(CloudFileSystemLifecycleState.Stopping, root.FileSystem.LifecycleState); + } + finally + { + root.Faults.BeforeCommit = null; + root.Faults.AfterCommit = null; + source.Release.TrySetResult(); + blocker.Dispose(); + } + + await feed.DisposeCompletion.WaitAsync(TimeSpan.FromSeconds(10)); + await root.RestartAsync(); + await using ICloudStateTransaction reopened = await root.Store.BeginTransactionAsync(); + Assert.True(LocalChangeCheckpoint.Decode((await reopened.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + Assert.Equal(markerCommitFails ? 1 : 0, (await reopened.Operations.ListAsync(100)).Count); + } + + private sealed class TailRenameSource(string before, string after) : ILocalChangeSource + { + private Func _handler = null!; + internal TaskCompletionSource Stopping { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal CancellationToken Cancellation { get; private set; } + + public Task StartAsync(Func eventHandler, CancellationToken cancellationToken) + { + _handler = eventHandler; + Cancellation = cancellationToken; + return Task.CompletedTask; + } + + public async ValueTask DisposeAsync() + { + await _handler(new(LocalChangeSourceAction.RenamedOldName, before)); + await _handler(new(LocalChangeSourceAction.RenamedNewName, after)); + Stopping.TrySetResult(); + await Release.Task; + } + } +} From 42430f1dd3f95897a17728cbc24fccaf106cc415 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:45:28 +0800 Subject: [PATCH 24/40] fix(namespace): prepare fresh intents for repeated directory moves --- docs/local-change-feed.md | 7 ++ .../Persistence/CloudDirectoryMoveEvidence.cs | 34 ++++++-- .../CloudDirectoryMoveCycleTests.cs | 80 +++++++++++++++++++ 3 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveCycleTests.cs diff --git a/docs/local-change-feed.md b/docs/local-change-feed.md index 8db0ac9..56ed844 100644 --- a/docs/local-change-feed.md +++ b/docs/local-change-feed.md @@ -116,6 +116,13 @@ operations pending. Dispatch and acknowledge them only after their meaning has b Native changes racing the transaction can require reconciliation even after its projection committed; the committed operation retains its original ID and sequence. Native notifications remain advisory. +Renaming a directory, undoing that rename, and repeating the same path pair uses a new preparation +from the latest retained pre-rename provenance, including after runtime/store restart and for +case-only renames. The mutable path-pair index points to that new preparation; earlier proofs, +membership manifests, receipts, and journal rows remain unchanged. Duplicate completed observations +still replay the original preparation. An explicitly supplied historical proof cannot project a +later namespace transition using its old receipt. + For an existing partial projection, use the original source directory's `MoveToAsync` or the prepared-proof recovery described in [placeholders](placeholders.md). A matching path or remote ID alone cannot establish historical native ownership. diff --git a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs index 5c117b1..77b09d7 100644 --- a/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs +++ b/src/CfSharp/Persistence/CloudDirectoryMoveEvidence.cs @@ -35,17 +35,34 @@ internal static string IntentName(string source, string target) => "cfsharp/name string source, string target, CancellationToken cancellationToken) { CloudStateCheckpoint? checkpoint = await transaction.Checkpoints.GetAsync(IntentName(source, target), cancellationToken).ConfigureAwait(false); + CloudDirectoryMoveProof? indexed = null; if (checkpoint is not null) { - CloudDirectoryMoveProof indexed = CloudDirectoryMoveProof.Decode(checkpoint.Value.Span); - return CloudDirectoryStateProjection.SamePath(source, indexed.SourceRelativePath) && target == indexed.DestinationRelativePath - ? indexed : throw new InvalidDataException("The directory intent index identifies another move."); + indexed = CloudDirectoryMoveProof.Decode(checkpoint.Value.Span); + if (!CloudDirectoryStateProjection.SamePath(source, indexed.SourceRelativePath) || target != indexed.DestinationRelativePath) + { + throw new InvalidDataException("The directory intent index identifies another move."); + } + + CloudStateCheckpoint? receipt = await transaction.Checkpoints.GetAsync(ReceiptName(indexed.ProofId), cancellationToken).ConfigureAwait(false); + if (receipt is null) + { + return indexed; + } + + if (!receipt.Value.Span.SequenceEqual(indexed.Encode())) + { + throw new InvalidDataException("The completed directory intent has an inconsistent receipt."); + } + // An unfinished preparation always retries its original proof. A completed index + // may instead identify an earlier cycle through this path pair; look for retained + // pre-rename provenance before falling back to historical replay. } Guid scope = await GetScopeAsync(transaction, false, cancellationToken).ConfigureAwait(false); if (scope == Guid.Empty) { - return null; + return indexed; } CloudItemState? from = await transaction.Items.GetByRelativePathAsync(source, cancellationToken).ConfigureAwait(false); @@ -70,6 +87,13 @@ internal static string IntentName(string source, string target) => "cfsharp/name continue; } + if (indexed is not null && provenance.RelativePath == indexed.DestinationRelativePath) + { + // A duplicate completed notification still sees post-move provenance. Exact + // spelling matters for case-only renames: it is not evidence of another move. + continue; + } + CloudStateCheckpoint? manifest = await transaction.Checkpoints.GetAsync(CloudDirectoryProvenance.MembersName(item.ItemId), cancellationToken).ConfigureAwait(false); if (manifest is null) { @@ -97,7 +121,7 @@ await transaction.Checkpoints.GetAsync(MembersName(prepared.ProofId), cancellati return prepared; } - return null; + return indexed; } internal static async ValueTask?> AuthenticateAsync(ICloudStateTransaction transaction, diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCycleTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCycleTests.cs new file mode 100644 index 0000000..41f51b5 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCycleTests.cs @@ -0,0 +1,80 @@ +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task RenameUndoAndRenameAgainUsesNewProofAndPreservesCompletedHistory(bool restart, bool caseOnly) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudOperationJournalEntry original = await SeedPendingChildAsync(root); + string destination = caseOnly ? "docs" : "Moved"; + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, destination)); + await ProcessRenameAsync(root, "Docs", destination); + CloudDirectoryMoveProof first; + CloudStateCheckpoint[] history; + await using (ICloudStateTransaction read = await root.Store.BeginTransactionAsync()) + { + first = CloudDirectoryMoveProof.Decode((await read.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.IntentName("Docs", destination)))!.Value.Span); + history = (await read.Checkpoints.ListAsync($"cfsharp/namespace/preparations/{first.ProofId:N}")).ToArray(); + Assert.Equal(3, history.Length); + } + + // A duplicate completed observation must still replay its original proof, including + // case-only paths whose source and destination compare equal on Windows. + await ProcessRenameAsync(root, "Docs", destination); + await using (ICloudStateTransaction read = await root.Store.BeginTransactionAsync()) + { + Assert.Equal(first.ProofId, CloudDirectoryMoveProof.Decode((await read.Checkpoints.GetAsync( + CloudDirectoryMoveEvidence.IntentName("Docs", destination)))!.Value.Span).ProofId); + Assert.False(LocalChangeCheckpoint.Decode((await read.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + } + + Directory.Move(Path.Combine(root.RootPath, destination), Path.Combine(root.RootPath, "Docs")); + await ProcessRenameAsync(root, destination, "Docs"); + await AssertProjectedChildAsync(root, "Docs", original); + if (restart) + { + await root.RestartAsync(); + } + + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, destination)); + // The immutable old receipt still cannot project a later namespace transition. Only + // automatic intent selection from the latest retained provenance may prepare a new move. + CloudDirectoryMoveReconciliationResult historical = await root.FileSystem.GetDirectory("Docs").ReconcileMoveAsync(first); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Conflict, historical.Outcome); + Assert.True(historical.RequiresFullRescan); + Assert.False(historical.DurableProjectionCommitted); + await AssertProjectedChildAsync(root, "Docs", original); + + await ProcessRenameAsync(root, "Docs", destination); + await AssertProjectedChildAsync(root, destination, original); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + CloudDirectoryMoveProof latest = CloudDirectoryMoveProof.Decode((await verify.Checkpoints.GetAsync( + CloudDirectoryMoveEvidence.IntentName("Docs", destination)))!.Value.Span); + Assert.NotEqual(first.ProofId, latest.ProofId); + Assert.Equal(first.ExpectedBinding, latest.ExpectedBinding); + Assert.Equal(first.ExpectedPlaceholderIdentity.ToArray(), latest.ExpectedPlaceholderIdentity.ToArray()); + Assert.NotNull(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(latest.ProofId))); + foreach (CloudStateCheckpoint saved in history) + { + CloudStateCheckpoint retained = (await verify.Checkpoints.GetAsync(saved.Name))!; + Assert.Equal(saved.Value.ToArray(), retained.Value.ToArray()); + Assert.Equal(saved.UpdatedAt, retained.UpdatedAt); + } + + Assert.Empty(await verify.Checkpoints.ListAsync(CloudLocalChangeFeed.NamespaceObservationsPrefix)); + Assert.False(LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + CloudOperationJournalEntry[] moves = (await verify.Operations.ListAsync(100)).Where(row => row.Kind == CloudStateOperationKind.Move).ToArray(); + Assert.Equal(4, moves.Length); + Assert.All(moves, move => Assert.Equal(root.DirectoryIdentity.ItemId, move.ItemId)); + Assert.Equal("Docs", LocalChangePayload.Decode(moves[0].Payload).PreviousRelativePath); + Assert.Equal(destination, LocalChangePayload.Decode(moves[0].Payload).RelativePath); + } +} From 774e691358869554e252be242ee00cbff270e8b4 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:06:10 +0800 Subject: [PATCH 25/40] test(sqlite): observe durable boundary before replay restart --- .../CloudLocalChangeFeedTests.cs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs index 763c663..2d00777 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeFeedTests.cs @@ -280,6 +280,7 @@ public async Task BoundedJournalReplaySurvivesFeedRestart() await using ICloudStateStore store = await OpenStoreAsync(); int acknowledged = 0; + CloudLocalChange[] retainedBeforeRestart; FakeSource source = new(); await using (CloudLocalChangeFeed feed = CloudLocalChangeFeed.CreateForTesting( _syncRootPath, @@ -298,6 +299,28 @@ public async Task BoundedJournalReplaySurvivesFeedRestart() Assert.NotEmpty(batch.Changes); acknowledged = batch.Changes.Count; await feed.AcknowledgeAsync(batch.Changes.Select(change => change.OperationId), timeout.Token); + + // This test verifies bounded durable replay, not a five-second flush of a queued + // burst. Observe the finite committed boundary before closing the first runtime. + using CancellationTokenSource persistenceTimeout = new(TimeSpan.FromSeconds(30)); + CloudLocalChangeScan scan; + do + { + scan = await feed.BeginScanAsync(persistenceTimeout.Token); + Assert.False(scan.RequiresFullRescan); + if (scan.ThroughSequence < eventsBeforeRestart) + { + await Task.Delay(10, persistenceTimeout.Token); + } + } + while (scan.ThroughSequence < eventsBeforeRestart); + + Assert.Equal(eventsBeforeRestart, scan.ThroughSequence); + CloudLocalChangePage retained = await feed.ReadPageAsync(scan, 0, eventCount, persistenceTimeout.Token); + Assert.False(retained.RequiresFullRescan); + Assert.False(retained.HasMore); + retainedBeforeRestart = retained.Changes.ToArray(); + Assert.Equal(eventsBeforeRestart - acknowledged, retainedBeforeRestart.Length); } FakeSource restartedSource = new(); @@ -307,6 +330,12 @@ public async Task BoundedJournalReplaySurvivesFeedRestart() options, restartedSource); await restarted.StartAsync(); + CloudLocalChangePage replayed = await restarted.ReadPageAsync(await restarted.BeginScanAsync(), 0, eventCount); + Assert.False(replayed.RequiresFullRescan); + Assert.Equal(retainedBeforeRestart.Select(change => (change.OperationId, change.Sequence, change.Kind, + change.ItemId, change.RelativePath, change.PreviousRelativePath, change.IsDirectory, change.ObservedAt)), + replayed.Changes.Select(change => (change.OperationId, change.Sequence, change.Kind, + change.ItemId, change.RelativePath, change.PreviousRelativePath, change.IsDirectory, change.ObservedAt))); for (int index = eventsBeforeRestart; index < eventCount; index++) { await restartedSource.EmitAsync(new(LocalChangeSourceAction.Created, paths[index])); From 9742b84280fc26e804f769e8a3ec5f3b4e043691 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:07:26 +0800 Subject: [PATCH 26/40] fix(namespace): finish admitted directory moves during shutdown Reuse the admitted operation lease and validated destination for preparation, proof retry, and post-native reconciliation. Keep new public work rejected during Stopping. Cover both shutdown cuts with real Cloud Files and SQLite, with and without an owned feed, and verify durable state after reopening. --- docs/sync-root-lifecycle.md | 5 + ...loudFileSystem.DirectoryMovePreparation.cs | 6 + ...dFileSystem.DirectoryMoveReconciliation.cs | 30 +++- .../CloudFileSystem.NamespaceOperations.cs | 15 +- .../Items/CloudItem.NamespaceOperations.cs | 7 + .../CloudDirectoryMoveShutdownTests.cs | 150 ++++++++++++++++++ 6 files changed, 201 insertions(+), 12 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveShutdownTests.cs diff --git a/docs/sync-root-lifecycle.md b/docs/sync-root-lifecycle.md index 49a20de..912efb4 100644 --- a/docs/sync-root-lifecycle.md +++ b/docs/sync-root-lifecycle.md @@ -34,6 +34,11 @@ state. Accepted renames retain internal path-scope admission during `Stopping`; hold the lifecycle admission lock while awaiting their completion. Concurrent disposal attempts remain serialized. It intentionally leaves the sync-root registration installed. +An admitted `MoveToAsync` retains the same operation lease, validated destination reference, and +state store through directory preparation and recovery. If disposal starts after preparation or +native movement, it waits for the admitted durable projection and native validation to finish. +Public operations remain closed during `Stopping`, including calls on existing item references. + ## Unregister only for removal Call `CloudSyncRoot.Unregister()` only when removing an account or uninstalling the provider. Windows diff --git a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs index 0e1eb40..270cb97 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs @@ -32,6 +32,12 @@ internal async ValueTask PrepareDirectoryMoveAsync(Clou throw new DirectoryNotFoundException("The destination parent does not exist."); } + return await PrepareDirectoryMoveCoreAsync(source, target, operation, cancellationToken).ConfigureAwait(false); + } + + private async ValueTask PrepareDirectoryMoveCoreAsync(CloudDirectory source, + CloudDirectory target, CloudFileSystemOperationLease operation, CancellationToken cancellationToken) + { // A metadata-only no-delete guard stabilizes this object during preparation. It does not // freeze descendant writes or placeholder identity changes and is never CFAPI protection. using SafeFileHandle guard = WindowsFileMetadata.Open(source.FullPath, preventDelete: true); diff --git a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs index 52e206b..e8cf28c 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMoveReconciliation.cs @@ -17,18 +17,36 @@ internal async ValueTask ReconcileDirect throw new ArgumentException("Recovery must be called on the original source reference.", nameof(proof)); } - CloudDirectoryMoveReconciliationStage stage = CloudDirectoryMoveReconciliationStage.Acquisition; - bool observed = false; - bool committed = false; - int updatedCount = 0; - string? durableRootPath = null; try { CloudDirectory target = GetDirectory(proof.DestinationRelativePath); using CloudFileSystemOperationLease operation = await AcquireOperationAsync( [CloudItemOperationScope.Subtree(source.FullPath), CloudItemOperationScope.Subtree(target.FullPath)], cancellationToken).ConfigureAwait(false); - stage = CloudDirectoryMoveReconciliationStage.PreparationValidation; + return await ReconcileDirectoryMoveCoreAsync(source, target, proof, operation, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException exception) + { + return new(CloudDirectoryMoveReconciliationOutcome.Canceled, CloudDirectoryMoveReconciliationStage.Acquisition, + false, false, false, exception, null); + } + catch (Exception exception) + { + return new(CloudDirectoryMoveReconciliationOutcome.Failed, CloudDirectoryMoveReconciliationStage.Acquisition, + false, false, false, exception, null); + } + } + + private async ValueTask ReconcileDirectoryMoveCoreAsync(CloudDirectory source, + CloudDirectory target, CloudDirectoryMoveProof proof, CloudFileSystemOperationLease operation, CancellationToken cancellationToken) + { + CloudDirectoryMoveReconciliationStage stage = CloudDirectoryMoveReconciliationStage.PreparationValidation; + bool observed = false; + bool committed = false; + int updatedCount = 0; + string? durableRootPath = null; + try + { await using (ICloudStateTransaction authentication = await operation.StateStore.BeginTransactionAsync(cancellationToken).ConfigureAwait(false)) { if (await CloudDirectoryMoveEvidence.AuthenticateAsync(authentication, proof, cancellationToken).ConfigureAwait(false) is null) diff --git a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs index 1d0b868..ce90c89 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs @@ -83,8 +83,8 @@ internal async ValueTask MoveAsync( throw new FileNotFoundException("Directory recovery requires original durable pre-move evidence.", item.FullPath); } - CloudDirectoryMoveReconciliationResult recovery = await ReconcileDirectoryMoveAsync(directory, - directoryProof, cancellationToken).ConfigureAwait(false); + CloudDirectoryMoveReconciliationResult recovery = await ReconcileDirectoryMoveCoreAsync(directory, + (CloudDirectory)movedItem, directoryProof, operation, cancellationToken).ConfigureAwait(false); if (IsCompletedDirectoryRecovery(recovery)) { return await CreateRecoveredMoveResultAsync(item, movedItem, operation.StateStore, recovery).ConfigureAwait(false); @@ -179,8 +179,8 @@ await PersistMovedStateAsync( { try { - directoryProof = await PrepareDirectoryMoveAsync(managedDirectory, destination, - validatedName, cancellationToken).ConfigureAwait(false); + directoryProof = await PrepareDirectoryMoveCoreAsync(managedDirectory, (CloudDirectory)movedItem, + operation, cancellationToken).ConfigureAwait(false); } catch (CloudDirectoryEvidenceUnavailableException) { @@ -207,8 +207,11 @@ await PersistMovedStateAsync( if (item is CloudDirectory movedDirectory && directoryProof is not null) { - CloudDirectoryMoveReconciliationResult recovery = await ReconcileDirectoryMoveAsync(movedDirectory, - directoryProof, CancellationToken.None).ConfigureAwait(false); + // Native success must finish its durable phase under the already admitted lease. + // Public reference creation or reacquisition would reject Stopping while disposal + // is waiting for this operation to release the still-live store and path scopes. + CloudDirectoryMoveReconciliationResult recovery = await ReconcileDirectoryMoveCoreAsync(movedDirectory, + (CloudDirectory)movedItem, directoryProof, operation, CancellationToken.None).ConfigureAwait(false); if (!IsCompletedDirectoryRecovery(recovery)) { ThrowDirectoryRecoveryFailure(item, movedItem, recovery, nativeMoveIssued: true, CancellationToken.None); diff --git a/src/CfSharp/Items/CloudItem.NamespaceOperations.cs b/src/CfSharp/Items/CloudItem.NamespaceOperations.cs index f5d5738..d473423 100644 --- a/src/CfSharp/Items/CloudItem.NamespaceOperations.cs +++ b/src/CfSharp/Items/CloudItem.NamespaceOperations.cs @@ -17,6 +17,12 @@ public abstract partial class CloudItem /// moving; a retry uses that retained evidence or the explicitly supplied proof and never moves /// an absent source again. An absent directory without historical evidence fails closed. /// + /// + /// An admitted move retains its operation lease through preparation, native movement, and + /// durable recovery. Concurrent owner disposal waits for that work; new public operations + /// are rejected after stopping begins. Cancellation cannot interrupt durable recovery after + /// the synchronous native move succeeds. + /// /// /// The destination belongs to another file system, the name is invalid, or replacement was /// requested for a directory. @@ -33,6 +39,7 @@ public abstract partial class CloudItem /// /// Cancellation was observed before the synchronous move or during pre-move state access. /// + /// The owner began stopping before this move was admitted. public ValueTask MoveToAsync( CloudDirectory destination, string name, diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveShutdownTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveShutdownTests.cs new file mode 100644 index 0000000..bc5ba5c --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveShutdownTests.cs @@ -0,0 +1,150 @@ +using System.Reflection; +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task AdmittedMoveCompletesDurableProjectionDuringOwnerShutdown(bool afterNativeMove, bool ownedFeed) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudOperationJournalEntry original = await SeedPendingChildAsync(root); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectory destination = root.FileSystem.Root; + ShutdownMoveSource? producer = ownedFeed ? new() : null; + CloudLocalChangeFeed? feed = null; + if (producer is not null) + { + feed = root.FileSystem.CreateLocalChangeFeed(); + // Preserve the owner's real acquisition and disposal callbacks while controlling + // exactly when the final rename pair reaches the drain queue. + typeof(CloudLocalChangeFeed).GetField("_source", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(feed, producer); + await feed.StartAsync(); + } + + TaskCompletionSource reachedCut = new(TaskCreationOptions.RunContinuationsAsynchronously); + using ManualResetEventSlim releaseMove = new(); + void PauseMove() + { + root.Faults.BeforeCommit = null; + root.Faults.AfterCommit = null; + reachedCut.TrySetResult(); + Assert.True(releaseMove.Wait(TimeSpan.FromSeconds(20)), "The test did not release the admitted move."); + } + + if (afterNativeMove) + { + // Preparation is still committed first. This second cut is at projection commit, + // with the native rename already complete and its metadata guard held. + root.Faults.BeforeCommit = () => + { + if (Directory.Exists(Path.Combine(root.RootPath, "Moved"))) + { + PauseMove(); + } + }; + } + else + { + root.Faults.AfterCommit = PauseMove; + } + + Task moveTask = Task.Run(async () => await source.MoveToAsync(destination, "Moved")); + Task? shutdown = null; + CloudItemMoveResult result; + try + { + await reachedCut.Task.WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal(afterNativeMove, Directory.Exists(Path.Combine(root.RootPath, "Moved"))); + Assert.Equal(!afterNativeMove, Directory.Exists(source.FullPath)); + shutdown = root.FileSystem.DisposeAsync().AsTask(); + Assert.True(SpinWait.SpinUntil(() => root.FileSystem.LifecycleState == CloudFileSystemLifecycleState.Stopping, + TimeSpan.FromSeconds(10))); + Assert.False(shutdown.IsCompleted); + Assert.Throws(() => root.FileSystem.GetDirectory("Moved")); + await Assert.ThrowsAsync(() => source.InspectAsync().AsTask()); + await Assert.ThrowsAsync(() => source.PrepareMoveAsync(destination, "Other").AsTask()); + if (producer is not null) + { + await producer.Stopping.Task.WaitAsync(TimeSpan.FromSeconds(10)); + } + + releaseMove.Set(); + result = await moveTask.WaitAsync(TimeSpan.FromSeconds(10)); + } + finally + { + root.Faults.BeforeCommit = null; + root.Faults.AfterCommit = null; + releaseMove.Set(); + if (shutdown is not null) + { + await shutdown.WaitAsync(TimeSpan.FromSeconds(15)); + } + + if (feed is not null) + { + await feed.DisposeCompletion.WaitAsync(TimeSpan.FromSeconds(10)); + } + } + + Assert.Equal(CloudFileSystemLifecycleState.Disposed, root.FileSystem.LifecycleState); + CloudDirectoryMoveReconciliationResult recovery = Assert.IsType(result.DirectoryReconciliation); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, recovery.Outcome); + Assert.Equal(CloudDirectoryMoveReconciliationStage.Completed, recovery.Stage); + Assert.True(recovery.NativeMoveObserved && recovery.DurableProjectionCommitted); + Assert.False(recovery.RequiresFullRescan); + Assert.Equal(2, result.DurableStateEntriesUpdated); + Assert.True(Directory.Exists(Path.Combine(root.RootPath, "Moved"))); + Assert.False(Directory.Exists(source.FullPath)); + + await root.RestartAsync(); + await AssertProjectedChildAsync(root, "Moved", original); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + CloudDirectoryMoveProof proof = CloudDirectoryMoveProof.Decode((await verify.Checkpoints.GetAsync( + CloudDirectoryMoveEvidence.IntentName("Docs", "Moved")))!.Value.Span); + Assert.NotNull(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + IReadOnlyList journal = await verify.Operations.ListAsync(100); + if (ownedFeed) + { + CloudOperationJournalEntry move = Assert.Single(journal.Where(row => row.Kind == CloudStateOperationKind.Move)); + Assert.Equal(root.DirectoryIdentity.ItemId, move.ItemId); + LocalChangePayload payload = LocalChangePayload.Decode(move.Payload); + Assert.Equal("Docs", payload.PreviousRelativePath); + Assert.Equal("Moved", payload.RelativePath); + Assert.True(payload.IsDirectory); + Assert.False(LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + } + else + { + Assert.Equal(original.OperationId, Assert.Single(journal).OperationId); + Assert.Null(await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName)); + } + } + + private sealed class ShutdownMoveSource : ILocalChangeSource + { + private Func _handler = null!; + internal TaskCompletionSource Stopping { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task StartAsync(Func eventHandler, CancellationToken cancellationToken) + { + _handler = eventHandler; + return Task.CompletedTask; + } + + public async ValueTask DisposeAsync() + { + await _handler(new(LocalChangeSourceAction.RenamedOldName, "Docs")); + await _handler(new(LocalChangeSourceAction.RenamedNewName, "Moved")); + Stopping.TrySetResult(); + } + } +} From 2877acd17a8268cc32fe73f5a7e41b2655222a2b Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:56:31 +0800 Subject: [PATCH 27/40] fix(local-changes): fence batches across persisted loss signals Capture the loss generation before each batch snapshot and recheck it after releasing the read transaction. A persisted loss marker cannot make an old checkpoint dispatchable. Cover overflow and watcher-error handoffs with real SQLite, preserve original journal fields, and verify explicit rescan recovery. --- docs/local-change-feed.md | 4 ++ .../LocalChanges/CloudLocalChangeFeed.cs | 11 ++- .../CloudLocalChangeScanFenceTests.cs | 71 +++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/docs/local-change-feed.md b/docs/local-change-feed.md index 56ed844..c76fb64 100644 --- a/docs/local-change-feed.md +++ b/docs/local-change-feed.md @@ -95,6 +95,10 @@ observations are checked in the same transaction as each page. Replay creation r acknowledging reconciliation. A loss received while the watcher worker waits for the store also blocks pages immediately. Reading a page never clears recovery markers. +Legacy batch reads also withhold their already-read backlog if loss arrives during the read, +even when the worker commits the rescan marker before the reading transaction returns. +The original journal remains pending until explicitly acknowledged after reconciliation. + An overflow/error or a full-rescan acknowledgement invalidates previously captured scans even after the durable marker is cleared. Begin a new scan after recovery. If a new loss races `AcknowledgeFullRescanAsync`, it either rejects acknowledgement with `InvalidOperationException` diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs index 4e06dd5..5900897 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs @@ -120,6 +120,11 @@ public async ValueTask StartAsync(CancellationToken cancellationToken = default) /// /// Token that cancels the read without acknowledging anything. /// An ordered bounded batch. The batch is immutable and safe to retain. + /// + /// Loss received during the journal read withholds changes and requires full reconciliation, + /// even if the worker persists its marker before the read finishes. Reading does not clear + /// that marker or acknowledge any pending operation. + /// /// The feed has not been started. /// The feed has been disposed. public async ValueTask ReadBatchAsync( @@ -129,6 +134,7 @@ public async ValueTask ReadBatchAsync( while (true) { ThrowIfFailed(); + long generation = Volatile.Read(ref _lossGeneration); await using ICloudStateTransaction transaction = await _stateStore .BeginTransactionAsync(cancellationToken).ConfigureAwait(false); LocalChangeCheckpoint checkpoint = await ReadCheckpointAsync(transaction, cancellationToken) @@ -138,7 +144,10 @@ public async ValueTask ReadBatchAsync( .ListAsync(_options.BatchSize, cancellationToken).ConfigureAwait(false); await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); ThrowIfFailed(); - bool rescan = creation || checkpoint.RequiresFullRescan || HasUnpersistedLoss; + // Persistence can clear HasUnpersistedLoss after this snapshot was read. Retain + // the generation boundary so that handoff cannot expose the earlier backlog. + bool rescan = creation || checkpoint.RequiresFullRescan || HasUnpersistedLoss || + generation != Volatile.Read(ref _lossGeneration); if (rescan && (creation || operations.Count != 0 || Interlocked.CompareExchange(ref _rescanNoticeDelivered, 1, 0) == 0)) { diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs index 1011904..929c89d 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs @@ -1,3 +1,6 @@ +using System.Reflection; +using System.Threading.Channels; + namespace CfSharp.Storage.Sqlite.Tests; public sealed partial class CloudLocalChangeFeedTests @@ -52,6 +55,74 @@ public async Task LossAtPageTransactionReleaseWithholdsAlreadyReadChanges() Assert.True(page.HasMore); } + [Theory] + [InlineData(false, 0)] + [InlineData(false, 2)] + [InlineData(true, 0)] + [InlineData(true, 2)] + public async Task PersistedLossAtBatchTransactionReleaseWithholdsBacklog(bool watcherError, int backlogSize) + { + await using ICloudStateStore store = await OpenStoreAsync(); + CloudOperationJournalEntry[] rows = await SeedChangesAsync(store, backlogSize); + RaceStore raceStore = new(store); + FakeSource source = new(); + await using CloudLocalChangeFeed feed = CreateFeed(raceStore, source); + await feed.StartAsync(); + Channel availability = (Channel)typeof(CloudLocalChangeFeed) + .GetField("_availability", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(feed)!; + Assert.False(availability.Reader.TryPeek(out _)); + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); + raceStore.AfterRollback = async () => + { + await source.EmitAsync(new(watcherError ? LocalChangeSourceAction.Error : LocalChangeSourceAction.Overflow, + string.Empty, watcherError ? 5 : 0)); + // The production worker publishes availability only after both committing its loss + // marker and marking that generation persisted. Preserve the signal for empty reads, + // but hold this older snapshot until the vulnerable handoff is complete. + Assert.True(await availability.Reader.WaitToReadAsync(timeout.Token)); + await using ICloudStateTransaction persisted = await store.BeginTransactionAsync(timeout.Token); + Assert.True(LocalChangeCheckpoint.Decode((await persisted.Checkpoints + .GetAsync(CloudLocalChangeFeed.CheckpointName, timeout.Token))!.Value).RequiresFullRescan); + await persisted.RollbackAsync(timeout.Token); + }; + + CloudLocalChangeBatch batch = await feed.ReadBatchAsync(timeout.Token); + Assert.True(batch.RequiresFullRescan); + Assert.Empty(batch.Changes); + await using (ICloudStateTransaction verify = await store.BeginTransactionAsync(timeout.Token)) + { + IReadOnlyList retained = await verify.Operations.ListAsync(10, timeout.Token); + Assert.Equal(rows.Length, retained.Count); + for (int index = 0; index < rows.Length; index++) + { + Assert.Equal(rows[index].OperationId, retained[index].OperationId); + Assert.Equal(rows[index].Sequence, retained[index].Sequence); + Assert.Equal(rows[index].Kind, retained[index].Kind); + Assert.Equal(rows[index].ItemId, retained[index].ItemId); + Assert.Equal(rows[index].Payload.ToArray(), retained[index].Payload.ToArray()); + Assert.Equal(rows[index].CreatedAt, retained[index].CreatedAt); + Assert.Equal(rows[index].AttemptCount, retained[index].AttemptCount); + Assert.Equal(rows[index].RetryAfter, retained[index].RetryAfter); + } + + await verify.RollbackAsync(timeout.Token); + } + + await feed.AcknowledgeFullRescanAsync(timeout.Token); + Assert.False((await feed.BeginScanAsync(timeout.Token)).RequiresFullRescan); + if (backlogSize != 0) + { + CloudLocalChangeBatch recovered = await feed.ReadBatchAsync(timeout.Token); + Assert.False(recovered.RequiresFullRescan); + Assert.Equal(rows.Select(row => row.OperationId), recovered.Changes.Select(change => change.OperationId)); + Assert.Equal(rows.Select(row => row.Sequence), recovered.Changes.Select(change => change.Sequence)); + Assert.Equal(rows.Select(row => LocalChangePayload.Decode(row.Payload).RelativePath), + recovered.Changes.Select(change => change.RelativePath)); + Assert.Equal(rows.Select(row => LocalChangePayload.Decode(row.Payload).ObservedAt), + recovered.Changes.Select(change => change.ObservedAt)); + } + } + [Fact] public async Task LossDuringRescanCommitCannotBeAcknowledgedByThatCommit() { From 828b5904036a4d6c1eea5388a7ac08f570529e18 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:34:15 +0800 Subject: [PATCH 28/40] fix(local-changes): capture durable loss boundaries for feed reads --- docs/local-change-feed.md | 4 +- .../CloudLocalChangeFeed.Scans.cs | 6 +- .../LocalChanges/CloudLocalChangeFeed.cs | 13 +-- .../CloudLocalChangeScanFenceTests.cs | 102 ++++++++++++++++-- 4 files changed, 106 insertions(+), 19 deletions(-) diff --git a/docs/local-change-feed.md b/docs/local-change-feed.md index c76fb64..a16acf1 100644 --- a/docs/local-change-feed.md +++ b/docs/local-change-feed.md @@ -95,8 +95,8 @@ observations are checked in the same transaction as each page. Replay creation r acknowledging reconciliation. A loss received while the watcher worker waits for the store also blocks pages immediately. Reading a page never clears recovery markers. -Legacy batch reads also withhold their already-read backlog if loss arrives during the read, -even when the worker commits the rescan marker before the reading transaction returns. +Legacy batch reads and scan capture retain loss already pending at entry or received during +the read, even when the worker commits the rescan marker before the reading transaction returns. The original journal remains pending until explicitly acknowledged after reconciliation. An overflow/error or a full-rescan acknowledgement invalidates previously captured scans even diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs index 27099ee..bba8194 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.Scans.cs @@ -15,6 +15,8 @@ public sealed partial class CloudLocalChangeFeed /// the captured boundary. Call ReadPageAsync with zero initially; restarting requires a new /// scan. This Windows feed can have concurrent consumers; callers own their dispatch and ACK /// coordination. Capturing a scan does not wait for native notifications still being processed. + /// Loss pending before or received during capture requires reconciliation even if its marker + /// finishes persisting before the scan is returned. /// /// The custom journal does not implement optional paging. /// The feed is not started or its worker failed. @@ -24,7 +26,9 @@ public async ValueTask BeginScanAsync(CancellationToken ca { EnsureStarted(); ThrowIfFailed(); - long generation = Volatile.Read(ref _lossGeneration); + // Match the snapshot against loss known durable before acquisition. A pending marker + // can finish after rollback, clearing the transient flag without belonging to this scan. + long generation = Volatile.Read(ref _persistedLossGeneration); long acknowledgementGeneration = Volatile.Read(ref _rescanAcknowledgementGeneration); await using ICloudStateTransaction transaction = await _stateStore.BeginTransactionAsync(cancellationToken) .ConfigureAwait(false); diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs index 5900897..78371c7 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs @@ -121,9 +121,9 @@ public async ValueTask StartAsync(CancellationToken cancellationToken = default) /// Token that cancels the read without acknowledging anything. /// An ordered bounded batch. The batch is immutable and safe to retain. /// - /// Loss received during the journal read withholds changes and requires full reconciliation, - /// even if the worker persists its marker before the read finishes. Reading does not clear - /// that marker or acknowledge any pending operation. + /// Loss pending before or received during the journal read withholds changes and requires + /// full reconciliation, even if the worker persists its marker before the read finishes. + /// Reading does not clear that marker or acknowledge any pending operation. /// /// The feed has not been started. /// The feed has been disposed. @@ -134,7 +134,7 @@ public async ValueTask ReadBatchAsync( while (true) { ThrowIfFailed(); - long generation = Volatile.Read(ref _lossGeneration); + long generation = Volatile.Read(ref _persistedLossGeneration); await using ICloudStateTransaction transaction = await _stateStore .BeginTransactionAsync(cancellationToken).ConfigureAwait(false); LocalChangeCheckpoint checkpoint = await ReadCheckpointAsync(transaction, cancellationToken) @@ -144,8 +144,9 @@ public async ValueTask ReadBatchAsync( .ListAsync(_options.BatchSize, cancellationToken).ConfigureAwait(false); await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); ThrowIfFailed(); - // Persistence can clear HasUnpersistedLoss after this snapshot was read. Retain - // the generation boundary so that handoff cannot expose the earlier backlog. + // Persistence can clear HasUnpersistedLoss after this snapshot was read. Capture + // the durable boundary before reading so loss already pending at entry is also + // retained across that handoff, rather than mistaken for part of this snapshot. bool rescan = creation || checkpoint.RequiresFullRescan || HasUnpersistedLoss || generation != Volatile.Read(ref _lossGeneration); if (rescan && (creation || operations.Count != 0 || diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs index 929c89d..b352cbd 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs @@ -1,5 +1,4 @@ using System.Reflection; -using System.Threading.Channels; namespace CfSharp.Storage.Sqlite.Tests; @@ -68,18 +67,14 @@ public async Task PersistedLossAtBatchTransactionReleaseWithholdsBacklog(bool wa FakeSource source = new(); await using CloudLocalChangeFeed feed = CreateFeed(raceStore, source); await feed.StartAsync(); - Channel availability = (Channel)typeof(CloudLocalChangeFeed) - .GetField("_availability", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(feed)!; - Assert.False(availability.Reader.TryPeek(out _)); using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); raceStore.AfterRollback = async () => { await source.EmitAsync(new(watcherError ? LocalChangeSourceAction.Error : LocalChangeSourceAction.Overflow, string.Empty, watcherError ? 5 : 0)); - // The production worker publishes availability only after both committing its loss - // marker and marking that generation persisted. Preserve the signal for empty reads, - // but hold this older snapshot until the vulnerable handoff is complete. - Assert.True(await availability.Reader.WaitToReadAsync(timeout.Token)); + // Hold the older snapshot until the worker has completed the persistence handoff. + // Availability is a consumable notification, not an exclusive completion barrier. + await WaitForLossPersistenceAsync(feed, timeout.Token); await using ICloudStateTransaction persisted = await store.BeginTransactionAsync(timeout.Token); Assert.True(LocalChangeCheckpoint.Decode((await persisted.Checkpoints .GetAsync(CloudLocalChangeFeed.CheckpointName, timeout.Token))!.Value).RequiresFullRescan); @@ -142,6 +137,64 @@ public async Task LossDuringRescanCommitCannotBeAcknowledgedByThatCommit() Assert.False((await feed.BeginScanAsync()).RequiresFullRescan); } + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task PendingLossBeforeReadCannotDisappearDuringSnapshotRelease(bool watcherError, bool scanRead) + { + await using ICloudStateStore store = await OpenStoreAsync(); + CloudOperationJournalEntry[] rows = await SeedChangesAsync(store, 2); + RaceStore raceStore = new(store); + FakeSource source = new(); + await using CloudLocalChangeFeed feed = CreateFeed(raceStore, source); + await feed.StartAsync(); + TaskCompletionSource lossPending = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource releasePersistence = new(TaskCreationOptions.RunContinuationsAsynchronously); + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); + raceStore.BeforeBegin = async () => + { + // MarkRescanRequired has recorded this loss generation, but cannot yet open its + // transaction. Let the reader acquire an older checkpoint before persistence. + lossPending.TrySetResult(); + await releasePersistence.Task.WaitAsync(timeout.Token); + }; + await source.EmitAsync(new(watcherError ? LocalChangeSourceAction.Error : LocalChangeSourceAction.Overflow, + string.Empty, watcherError ? 5 : 0)); + try + { + await lossPending.Task.WaitAsync(timeout.Token); + raceStore.AfterRollback = async () => + { + releasePersistence.TrySetResult(); + await WaitForLossPersistenceAsync(feed, timeout.Token); + }; + if (scanRead) + { + CloudLocalChangeScan scan = await feed.BeginScanAsync(timeout.Token); + Assert.True(scan.RequiresFullRescan); + Assert.Empty((await feed.ReadPageAsync(scan, 0, 2, timeout.Token)).Changes); + } + else + { + CloudLocalChangeBatch batch = await feed.ReadBatchAsync(timeout.Token); + Assert.True(batch.RequiresFullRescan); + Assert.Empty(batch.Changes); + } + + await feed.AcknowledgeFullRescanAsync(timeout.Token); + CloudLocalChangePage recovered = await feed.ReadPageAsync(await feed.BeginScanAsync(timeout.Token), 0, 2, timeout.Token); + Assert.False(recovered.RequiresFullRescan); + Assert.Equal(rows.Select(row => row.OperationId), recovered.Changes.Select(change => change.OperationId)); + Assert.Equal(rows.Select(row => row.Sequence), recovered.Changes.Select(change => change.Sequence)); + } + finally + { + releasePersistence.TrySetResult(); + } + } + [Fact] public async Task CreationIntentAndRetainedObservationFenceScansUntilExplicitRecovery() { @@ -205,6 +258,25 @@ await write.Checkpoints.UpsertAsync(new CloudStateCheckpoint(CloudLocalChangeFee Assert.True(page.RequiresFullRescan); } + private static async Task WaitForLossPersistenceAsync(CloudLocalChangeFeed feed, CancellationToken cancellationToken) + { + // These private counters select the race boundary only; assertions exercise the public + // feed and real durable journal. No timing delay substitutes for the worker's commit. + FieldInfo loss = typeof(CloudLocalChangeFeed).GetField("_lossGeneration", BindingFlags.NonPublic | BindingFlags.Instance)!; + FieldInfo persisted = typeof(CloudLocalChangeFeed).GetField("_persistedLossGeneration", BindingFlags.NonPublic | BindingFlags.Instance)!; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + long generation = (long)loss.GetValue(feed)!; + if (generation != 0 && generation == (long)persisted.GetValue(feed)!) + { + return; + } + + await Task.Delay(10, cancellationToken); + } + } + private static async Task AcknowledgePersistedRescanAsync(CloudLocalChangeFeed feed) { using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); @@ -224,10 +296,20 @@ private static async Task AcknowledgePersistedRescanAsync(CloudLocalChangeFeed f private sealed class RaceStore(ICloudStateStore inner) : ICloudStateStore { + internal Func? BeforeBegin { get; set; } internal Func? AfterRollback { get; set; } internal Func? BeforeCommit { get; set; } - public async ValueTask BeginTransactionAsync(CancellationToken cancellationToken = default) => - new RaceTransaction(await inner.BeginTransactionAsync(cancellationToken), this); + public async ValueTask BeginTransactionAsync(CancellationToken cancellationToken = default) + { + Func? hook = BeforeBegin; + BeforeBegin = null; + if (hook is not null) + { + await hook(); + } + + return new RaceTransaction(await inner.BeginTransactionAsync(cancellationToken), this); + } public ValueTask DisposeAsync() => ValueTask.CompletedTask; } From ec0f005febec78cdb8decad0a6487438fcf3754f Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:35:49 +0800 Subject: [PATCH 29/40] test(integration): verify reference protection across sharing conflicts --- ...loudContentConfirmationConcurrencyTests.cs | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/tests/CfSharp.IntegrationTests/CloudContentConfirmationConcurrencyTests.cs b/tests/CfSharp.IntegrationTests/CloudContentConfirmationConcurrencyTests.cs index 0997f88..0fdf8d5 100644 --- a/tests/CfSharp.IntegrationTests/CloudContentConfirmationConcurrencyTests.cs +++ b/tests/CfSharp.IntegrationTests/CloudContentConfirmationConcurrencyTests.cs @@ -87,7 +87,7 @@ public void Dispose() [InlineData("write")] [InlineData("rename")] [InlineData("replace")] - public async Task IndependentProcessWaitsForReferenceAndContinuesAfterRelease(string action) + public async Task IndependentProcessCannotMutateUntilReferenceRelease(string action) { await using Fixture fixture = await Fixture.StartAsync(); byte[] content = "upload"u8.ToArray(); @@ -109,13 +109,40 @@ public async Task IndependentProcessWaitsForReferenceAndContinuesAfterRelease(st await WaitForReadyAsync(competitor); competitor.Go.Set(); await Task.Delay(100); - Assert.True(!competitor.HasExited, + // Some Cloud Files states reject the write open with ERROR_SHARING_VIOLATION + // instead of waiting for an oplock break. Both outcomes prevent mutation. Retry + // only an observed pre-release sharing conflict, while the original lease lives. + bool sharingConflict = action == "write" && competitor.HasExited && competitor.ExitCode == 32; + Assert.True(sharingConflict || !competitor.HasExited, $"Competitor completed while the reference was held: action={action}; exit={(competitor.HasExited ? competitor.ExitCode : null)}"); long released = Stopwatch.GetTimestamp(); await reference.DisposeAsync(); - await competitor.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10)); - Assert.True(competitor.ExitCode == 0, $"Competitor exit={competitor.ExitCode}; Native=0x{competitor.ExitCode:X8}"); - output.WriteLine($"Competitor={action}; WaitAfterReleaseMs={Stopwatch.GetElapsedTime(released).TotalMilliseconds}"); + if (sharingConflict) + { + using Competitor retry = StartCompetitor(fixture.File.FullPath, target, action); + try + { + await WaitForReadyAsync(retry); + retry.Go.Set(); + await retry.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal(0, retry.ExitCode); + } + finally + { + if (!retry.HasExited) + { + retry.Kill(entireProcessTree: true); + await retry.WaitForExitAsync(); + } + } + } + else + { + await competitor.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10)); + Assert.True(competitor.ExitCode == 0, $"Competitor exit={competitor.ExitCode}; Native=0x{competitor.ExitCode:X8}"); + } + + output.WriteLine($"Competitor={action}; PreReleaseSharingConflict={sharingConflict}; WaitAfterReleaseMs={Stopwatch.GetElapsedTime(released).TotalMilliseconds}"); await Assert.ThrowsAsync(() => lease.BeginTransferAsync().AsTask()); await lease.DisposeAsync(); if (action == "write") From 05b305645407702643f39826327fa3ef4ea4706b Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:03:55 +0800 Subject: [PATCH 30/40] fix(local-changes): detach source consumption from caller context --- docs/local-change-feed.md | 2 + .../LocalChanges/CloudLocalChangeFeed.cs | 5 +- .../CloudLocalChangeScanFenceTests.cs | 80 +++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/docs/local-change-feed.md b/docs/local-change-feed.md index a16acf1..bd7e1b5 100644 --- a/docs/local-change-feed.md +++ b/docs/local-change-feed.md @@ -34,6 +34,8 @@ else The feed normalizes paths relative to the sync root, pairs renames, and persists the ordered journal with its watcher checkpoint in one transaction. Acknowledgement advances durable progress only after the application has accepted the batch. +Background notification processing does not require pumping the synchronization context that +started the feed; loss persistence and shutdown drain continue independently of that context. ## Scan a finite backlog without acknowledging its head diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs index 78371c7..2bb2051 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs @@ -86,6 +86,7 @@ internal CloudLocalChangeFeed( /// Starts the native watcher and the durable normalization worker. /// Token that cancels startup before the watcher is opened. + /// Background notification consumption does not require pumping the starting synchronization context. /// The feed has already been started. /// The feed has been disposed. public async ValueTask StartAsync(CancellationToken cancellationToken = default) @@ -515,7 +516,9 @@ private async Task ProcessEventsAsync() { try { - await foreach (LocalChangeSourceEvent change in _sourceEvents.Reader.ReadAllAsync(_shutdown.Token)) + // The processor outlives StartAsync. Its next notification must not depend on the + // caller's UI/test context remaining available to commit loss or finish draining. + await foreach (LocalChangeSourceEvent change in _sourceEvents.Reader.ReadAllAsync(_shutdown.Token).ConfigureAwait(false)) { await ProcessEventAsync(change, _shutdown.Token).ConfigureAwait(false); if (Interlocked.Exchange(ref _overflowSignaled, 0) != 0) diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs index b352cbd..12a0eb4 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs @@ -4,6 +4,51 @@ namespace CfSharp.Storage.Sqlite.Tests; public sealed partial class CloudLocalChangeFeedTests { + [Fact] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "xUnit1031:Do not use blocking task operations in test method", + Justification = "A dedicated background caller installs a paused context; the test awaits startup and rescues queued callbacks.")] + public async Task SourceWorkerPersistsLossWithoutPumpingTheStartingContext() + { + await using ICloudStateStore store = await OpenStoreAsync(); + FakeSource source = new(); + await using CloudLocalChangeFeed feed = CreateFeed(store, source); + PausedSourceContext context = new(); + TaskCompletionSource started = new(TaskCreationOptions.RunContinuationsAsynchronously); + Thread caller = new(() => + { + SynchronizationContext.SetSynchronizationContext(context); + try + { + feed.StartAsync().AsTask().GetAwaiter().GetResult(); + started.TrySetResult(); + } + catch (Exception exception) { started.TrySetException(exception); } + finally { SynchronizationContext.SetSynchronizationContext(null); } + }) + { IsBackground = true }; + caller.Start(); + try + { + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(caller.Join(TimeSpan.FromSeconds(5))); + await source.EmitAsync(new(LocalChangeSourceAction.Overflow, string.Empty)); + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(5)); + bool persistedWithoutPumping = true; + try { await WaitForLossPersistenceAsync(feed, timeout.Token); } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) { persistedWithoutPumping = false; } + Assert.True(persistedWithoutPumping, $"Loss persistence required pumping the startup context; queued callbacks={context.Posts}."); + Assert.Equal(0, context.Posts); + await using ICloudStateTransaction verify = await store.BeginTransactionAsync(timeout.Token); + Assert.True(LocalChangeCheckpoint.Decode((await verify.Checkpoints + .GetAsync(CloudLocalChangeFeed.CheckpointName, timeout.Token))!.Value).RequiresFullRescan); + } + finally + { + // Rescue the baseline worker before disposal, rather than leak a blocked processor. + context.Release(); + } + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -294,6 +339,41 @@ private static async Task AcknowledgePersistedRescanAsync(CloudLocalChangeFeed f } } + private sealed class PausedSourceContext : SynchronizationContext + { + private readonly object _gate = new(); + private readonly Queue<(SendOrPostCallback Callback, object? State)> _pending = new(); + private bool _released; + private int _posts; + internal int Posts => Volatile.Read(ref _posts); + public override void Post(SendOrPostCallback callback, object? state) + { + Interlocked.Increment(ref _posts); + lock (_gate) + { + if (!_released) + { + _pending.Enqueue((callback, state)); + return; + } + } + + ThreadPool.QueueUserWorkItem(_ => callback(state)); + } + + internal void Release() + { + lock (_gate) + { + _released = true; + while (_pending.TryDequeue(out var continuation)) + { + ThreadPool.QueueUserWorkItem(_ => continuation.Callback(continuation.State)); + } + } + } + } + private sealed class RaceStore(ICloudStateStore inner) : ICloudStateStore { internal Func? BeforeBegin { get; set; } From 7b4c33a040c09b79b95f2f3865afee19ad2550e1 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:34:27 +0800 Subject: [PATCH 31/40] test(local-changes): detach persistence barriers from caller context --- .../CloudLocalChangeScanFenceTests.cs | 65 ++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs index 12a0eb4..130b3d6 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs @@ -104,6 +104,8 @@ public async Task LossAtPageTransactionReleaseWithholdsAlreadyReadChanges() [InlineData(false, 2)] [InlineData(true, 0)] [InlineData(true, 2)] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "xUnit1030:Do not use ConfigureAwait(false) in test method", + Justification = "Only store callbacks detach from the deliberately paused context; test orchestration retains the xUnit context.")] public async Task PersistedLossAtBatchTransactionReleaseWithholdsBacklog(bool watcherError, int backlogSize) { await using ICloudStateStore store = await OpenStoreAsync(); @@ -113,20 +115,51 @@ public async Task PersistedLossAtBatchTransactionReleaseWithholdsBacklog(bool wa await using CloudLocalChangeFeed feed = CreateFeed(raceStore, source); await feed.StartAsync(); using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); + TaskCompletionSource releaseMarker = new(TaskCreationOptions.RunContinuationsAsynchronously); raceStore.AfterRollback = async () => { + raceStore.BeforeBegin = () => new ValueTask(releaseMarker.Task.WaitAsync(timeout.Token)); await source.EmitAsync(new(watcherError ? LocalChangeSourceAction.Error : LocalChangeSourceAction.Overflow, - string.Empty, watcherError ? 5 : 0)); + string.Empty, watcherError ? 5 : 0)).ConfigureAwait(false); // Hold the older snapshot until the worker has completed the persistence handoff. // Availability is a consumable notification, not an exclusive completion barrier. - await WaitForLossPersistenceAsync(feed, timeout.Token); - await using ICloudStateTransaction persisted = await store.BeginTransactionAsync(timeout.Token); + Task persistence = WaitForLossPersistenceAsync(feed, timeout.Token); + releaseMarker.TrySetResult(); + await persistence.ConfigureAwait(false); + await using ICloudStateTransaction persisted = await store.BeginTransactionAsync(timeout.Token).ConfigureAwait(false); Assert.True(LocalChangeCheckpoint.Decode((await persisted.Checkpoints - .GetAsync(CloudLocalChangeFeed.CheckpointName, timeout.Token))!.Value).RequiresFullRescan); - await persisted.RollbackAsync(timeout.Token); + .GetAsync(CloudLocalChangeFeed.CheckpointName, timeout.Token).ConfigureAwait(false))!.Value).RequiresFullRescan); + await persisted.RollbackAsync(timeout.Token).ConfigureAwait(false); }; - CloudLocalChangeBatch batch = await feed.ReadBatchAsync(timeout.Token); + // Force the test barrier to begin waiting before the worker commits, under a context + // that will never be pumped. Test-store wrappers must not capture the caller either. + PausedSourceContext context = new(); + SynchronizationContext? previous = SynchronizationContext.Current; + Task read; + SynchronizationContext.SetSynchronizationContext(context); + try { read = feed.ReadBatchAsync(timeout.Token).AsTask(); } + finally { SynchronizationContext.SetSynchronizationContext(previous); } + CloudLocalChangeBatch batch; + try + { + batch = await read.WaitAsync(timeout.Token); + Assert.Equal(0, context.Posts); + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) + { + long loss = (long)typeof(CloudLocalChangeFeed).GetField("_lossGeneration", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(feed)!; + long persisted = (long)typeof(CloudLocalChangeFeed).GetField("_persistedLossGeneration", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(feed)!; + throw new TimeoutException($"Test persistence barrier did not complete; context posts={context.Posts}; loss={loss}; persisted={persisted}."); + } + finally + { + releaseMarker.TrySetResult(); + context.Release(); + try { await read; } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) { } + } + Assert.True(batch.RequiresFullRescan); Assert.Empty(batch.Changes); await using (ICloudStateTransaction verify = await store.BeginTransactionAsync(timeout.Token)) @@ -187,6 +220,8 @@ public async Task LossDuringRescanCommitCannotBeAcknowledgedByThatCommit() [InlineData(false, true)] [InlineData(true, false)] [InlineData(true, true)] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "xUnit1030:Do not use ConfigureAwait(false) in test method", + Justification = "Only worker/store barrier callbacks detach, matching the real asynchronous store; test orchestration retains the xUnit context.")] public async Task PendingLossBeforeReadCannotDisappearDuringSnapshotRelease(bool watcherError, bool scanRead) { await using ICloudStateStore store = await OpenStoreAsync(); @@ -203,7 +238,7 @@ public async Task PendingLossBeforeReadCannotDisappearDuringSnapshotRelease(bool // MarkRescanRequired has recorded this loss generation, but cannot yet open its // transaction. Let the reader acquire an older checkpoint before persistence. lossPending.TrySetResult(); - await releasePersistence.Task.WaitAsync(timeout.Token); + await releasePersistence.Task.WaitAsync(timeout.Token).ConfigureAwait(false); }; await source.EmitAsync(new(watcherError ? LocalChangeSourceAction.Error : LocalChangeSourceAction.Overflow, string.Empty, watcherError ? 5 : 0)); @@ -213,7 +248,7 @@ await source.EmitAsync(new(watcherError ? LocalChangeSourceAction.Error : LocalC raceStore.AfterRollback = async () => { releasePersistence.TrySetResult(); - await WaitForLossPersistenceAsync(feed, timeout.Token); + await WaitForLossPersistenceAsync(feed, timeout.Token).ConfigureAwait(false); }; if (scanRead) { @@ -318,7 +353,7 @@ private static async Task WaitForLossPersistenceAsync(CloudLocalChangeFeed feed, return; } - await Task.Delay(10, cancellationToken); + await Task.Delay(10, cancellationToken).ConfigureAwait(false); } } @@ -385,10 +420,10 @@ public async ValueTask BeginTransactionAsync(Cancellatio BeforeBegin = null; if (hook is not null) { - await hook(); + await hook().ConfigureAwait(false); } - return new RaceTransaction(await inner.BeginTransactionAsync(cancellationToken), this); + return new RaceTransaction(await inner.BeginTransactionAsync(cancellationToken).ConfigureAwait(false), this); } public ValueTask DisposeAsync() => ValueTask.CompletedTask; } @@ -407,20 +442,20 @@ public async ValueTask CommitAsync(CancellationToken cancellationToken = default store.BeforeCommit = null; if (hook is not null) { - await hook(); + await hook().ConfigureAwait(false); } - await inner.CommitAsync(cancellationToken); + await inner.CommitAsync(cancellationToken).ConfigureAwait(false); } public async ValueTask RollbackAsync(CancellationToken cancellationToken = default) { - await inner.RollbackAsync(cancellationToken); + await inner.RollbackAsync(cancellationToken).ConfigureAwait(false); Func? hook = store.AfterRollback; store.AfterRollback = null; if (hook is not null) { - await hook(); + await hook().ConfigureAwait(false); } } From a37ed4e63acf1596f4b3214ef92d25d600c81e3e Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:59:51 +0800 Subject: [PATCH 32/40] test(local-changes): scope cancellation budgets to verification phases --- .../CloudLocalChangeScanFenceTests.cs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs index 130b3d6..bafacc0 100644 --- a/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs +++ b/tests/CfSharp.Storage.Sqlite.Tests/CloudLocalChangeScanFenceTests.cs @@ -162,9 +162,14 @@ await source.EmitAsync(new(watcherError ? LocalChangeSourceAction.Error : LocalC Assert.True(batch.RequiresFullRescan); Assert.Empty(batch.Changes); - await using (ICloudStateTransaction verify = await store.BeginTransactionAsync(timeout.Token)) + // The read budget ends with its completed handoff. A queued test-runner continuation + // may resume later; durable verification must not reuse that expired operation token. + timeout.Cancel(); + using CancellationTokenSource verificationTimeout = new(TimeSpan.FromSeconds(10)); + CancellationToken verificationToken = verificationTimeout.Token; + await using (ICloudStateTransaction verify = await store.BeginTransactionAsync(verificationToken)) { - IReadOnlyList retained = await verify.Operations.ListAsync(10, timeout.Token); + IReadOnlyList retained = await verify.Operations.ListAsync(10, verificationToken); Assert.Equal(rows.Length, retained.Count); for (int index = 0; index < rows.Length; index++) { @@ -178,14 +183,14 @@ await source.EmitAsync(new(watcherError ? LocalChangeSourceAction.Error : LocalC Assert.Equal(rows[index].RetryAfter, retained[index].RetryAfter); } - await verify.RollbackAsync(timeout.Token); + await verify.RollbackAsync(verificationToken); } - await feed.AcknowledgeFullRescanAsync(timeout.Token); - Assert.False((await feed.BeginScanAsync(timeout.Token)).RequiresFullRescan); + await feed.AcknowledgeFullRescanAsync(verificationToken); + Assert.False((await feed.BeginScanAsync(verificationToken)).RequiresFullRescan); if (backlogSize != 0) { - CloudLocalChangeBatch recovered = await feed.ReadBatchAsync(timeout.Token); + CloudLocalChangeBatch recovered = await feed.ReadBatchAsync(verificationToken); Assert.False(recovered.RequiresFullRescan); Assert.Equal(rows.Select(row => row.OperationId), recovered.Changes.Select(change => change.OperationId)); Assert.Equal(rows.Select(row => row.Sequence), recovered.Changes.Select(change => change.Sequence)); From b9926eabb822bcb339cdc7b0989bfee9e205b18d Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:15:02 +0800 Subject: [PATCH 33/40] perf(persistence): refresh directory membership once per batch --- docs/placeholders.md | 5 +- .../CloudFileSystem.PlaceholderOperations.cs | 7 +- .../Persistence/CloudDirectoryProvenance.cs | 100 ++++++--- src/CfSharp/Providers/CloudProviderSession.cs | 9 +- .../CloudDirectoryBatchProvenanceTests.cs | 192 ++++++++++++++++++ .../CloudDirectoryMoveTests.cs | 44 +++- 6 files changed, 310 insertions(+), 47 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryBatchProvenanceTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index c467cb1..389069c 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -16,7 +16,10 @@ complete native IDs. Metadata capture reads no content and holds no continuing l Directory creation, conversion, identity replacement, remote creation, and provider population retain native provenance in the same transaction as their official item rows. Adding children -refreshes known membership of captured ancestors without enumerating native subtrees. Immutable +refreshes known membership of captured ancestors without enumerating native subtrees. Batch creation +and each provider population page capture new directory bindings first, then refresh each affected +directory's membership once from all item rows in the same transaction, including shared ancestors. +The existing recovery metadata limits still apply. Immutable prepared proofs remain unchanged. Removing an identity or reverting a directory removes its live provenance. Storage without complete IDs retains existing creation behavior but cannot establish this recovery evidence; legacy directories can explicitly prepare while still at their source. diff --git a/src/CfSharp/FileSystem/CloudFileSystem.PlaceholderOperations.cs b/src/CfSharp/FileSystem/CloudFileSystem.PlaceholderOperations.cs index dcc8426..f302772 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.PlaceholderOperations.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.PlaceholderOperations.cs @@ -308,11 +308,8 @@ await transaction.Items.UpsertAsync( CancellationToken.None).ConfigureAwait(false); } - foreach (PlaceholderCreationWorkEntry entry in entries) - { - await CloudDirectoryProvenance.RetainProjectionAsync(transaction, entry.Item.SyncRootPath, - entry.Item.RelativePath, entry.Specification.Kind, CancellationToken.None).ConfigureAwait(false); - } + await CloudDirectoryProvenance.RetainProjectionsAsync(transaction, entries[0].Item.SyncRootPath, + entries.Select(static entry => (entry.Item.RelativePath, entry.Specification.Kind)), CancellationToken.None).ConfigureAwait(false); await transaction.CommitAsync(CancellationToken.None).ConfigureAwait(false); } diff --git a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs index c8a6bb1..3c06f91 100644 --- a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs +++ b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs @@ -68,48 +68,54 @@ internal static CloudDirectoryProvenance Decode(ReadOnlyMemory value) } [SupportedOSPlatform("windows10.0.16299")] - internal static async ValueTask RetainProjectionAsync(ICloudStateTransaction transaction, string syncRootPath, - string relativePath, CloudItemKind kind, CancellationToken cancellationToken) + internal static ValueTask RetainProjectionAsync(ICloudStateTransaction transaction, string syncRootPath, + string relativePath, CloudItemKind kind, CancellationToken cancellationToken) => + RetainProjectionsAsync(transaction, syncRootPath, [(relativePath, kind)], cancellationToken); + + [SupportedOSPlatform("windows10.0.16299")] + internal static async ValueTask RetainProjectionsAsync(ICloudStateTransaction transaction, string syncRootPath, + IEnumerable<(string RelativePath, CloudItemKind Kind)> projections, CancellationToken cancellationToken) { - if (kind == CloudItemKind.Directory && relativePath.Length != 0) + Dictionary captured = new(StringComparer.OrdinalIgnoreCase); + HashSet ancestors = new(StringComparer.OrdinalIgnoreCase); + foreach ((string relativePath, CloudItemKind kind) in projections) { - using SafeFileHandle guard = WindowsFileMetadata.Open(Path.Combine(syncRootPath, relativePath), preventDelete: true); - DirectoryNativeObservation captured; - try - { - captured = CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), syncRootPath); - } - catch (Win32Exception exception) when (exception.NativeErrorCode is 1 or 50 or 87) + if (kind == CloudItemKind.Directory && relativePath.Length != 0 && !captured.ContainsKey(relativePath)) { - // Complete file IDs are optional for existing creation APIs. Do not fabricate - // recovery evidence on storage that cannot report them. - return; - } - catch (CloudDirectoryEvidenceUnavailableException) - { - return; + CloudDirectoryProvenance? provenance = await CaptureProjectionAsync(transaction, syncRootPath, + relativePath, cancellationToken).ConfigureAwait(false); + if (provenance is null) + { + continue; + } + + captured.Add(relativePath, provenance); } - CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Decode(captured.Identity); - CloudItemState? item = await transaction.Items.GetByItemIdAsync(identity.ItemId, cancellationToken).ConfigureAwait(false); - if (item is null || item.Kind != CloudItemKind.Directory || item.IsTombstone || item.RemoteId != identity.RemoteId || - !string.Equals(item.RelativePath, relativePath, StringComparison.OrdinalIgnoreCase) || - !string.Equals(captured.ActualPath, Path.Combine(syncRootPath, relativePath), StringComparison.OrdinalIgnoreCase)) + string? parent = Path.GetDirectoryName(relativePath); + while (!string.IsNullOrEmpty(parent) && ancestors.Add(parent)) { - throw new InvalidOperationException("Directory provenance does not match the projected native item."); + parent = Path.GetDirectoryName(parent); } + } - Guid scope = await CloudDirectoryMoveEvidence.GetScopeAsync(transaction, true, cancellationToken).ConfigureAwait(false); - CloudDirectoryProvenance provenance = new(Guid.NewGuid(), scope, item.ItemId, - Path.GetRelativePath(syncRootPath, captured.ActualPath), captured.Binding, captured.Identity); + // All item rows must already be written. Capture each new native binding first, then + // encode each affected directory's complete membership once in this transaction. A + // newly captured directory can also be another entry's ancestor; do not scan it twice. + foreach (CloudDirectoryProvenance provenance in captured.Values) + { await RetainMetadataAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); } - // Refresh known membership of already captured ancestors using official transactional - // rows, without traversing native subtrees or manufacturing a new historical binding. - string? parent = Path.GetDirectoryName(relativePath); - while (!string.IsNullOrEmpty(parent)) + // Refresh existing bindings without traversing native subtrees or manufacturing new + // historical evidence. Shared ancestors of siblings use the same final row snapshot. + foreach (string parent in ancestors) { + if (captured.ContainsKey(parent)) + { + continue; + } + CloudItemState? directory = await transaction.Items.GetByRelativePathAsync(parent, cancellationToken).ConfigureAwait(false); if (directory is { Kind: CloudItemKind.Directory, IsTombstone: false }) { @@ -124,9 +130,41 @@ internal static async ValueTask RetainProjectionAsync(ICloudStateTransaction tra } } } + } + } - parent = Path.GetDirectoryName(parent); + [SupportedOSPlatform("windows10.0.16299")] + private static async ValueTask CaptureProjectionAsync(ICloudStateTransaction transaction, + string syncRootPath, string relativePath, CancellationToken cancellationToken) + { + using SafeFileHandle guard = WindowsFileMetadata.Open(Path.Combine(syncRootPath, relativePath), preventDelete: true); + DirectoryNativeObservation captured; + try + { + captured = CloudDirectoryMoveEvidence.Capture(guard.DangerousGetHandle(), syncRootPath); + } + catch (Win32Exception exception) when (exception.NativeErrorCode is 1 or 50 or 87) + { + // Complete file IDs are optional for existing creation APIs. Do not fabricate + // recovery evidence on storage that cannot report them. + return null; } + catch (CloudDirectoryEvidenceUnavailableException) + { + return null; + } + + CloudPlaceholderIdentity identity = CloudPlaceholderIdentity.Decode(captured.Identity); + CloudItemState? item = await transaction.Items.GetByItemIdAsync(identity.ItemId, cancellationToken).ConfigureAwait(false); + if (item is null || item.Kind != CloudItemKind.Directory || item.IsTombstone || item.RemoteId != identity.RemoteId || + !string.Equals(item.RelativePath, relativePath, StringComparison.OrdinalIgnoreCase) || + !string.Equals(captured.ActualPath, Path.Combine(syncRootPath, relativePath), StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Directory provenance does not match the projected native item."); + } + + Guid scope = await CloudDirectoryMoveEvidence.GetScopeAsync(transaction, true, cancellationToken).ConfigureAwait(false); + return new(Guid.NewGuid(), scope, item.ItemId, Path.GetRelativePath(syncRootPath, captured.ActualPath), captured.Binding, captured.Identity); } internal static async ValueTask ProjectPathsAsync(ICloudStateTransaction transaction, CloudDirectoryMoveProof proof, diff --git a/src/CfSharp/Providers/CloudProviderSession.cs b/src/CfSharp/Providers/CloudProviderSession.cs index deca0d5..882d3bd 100644 --- a/src/CfSharp/Providers/CloudProviderSession.cs +++ b/src/CfSharp/Providers/CloudProviderSession.cs @@ -923,12 +923,9 @@ await transaction.Items.UpsertAsync( _shutdown.Token).ConfigureAwait(false); } - foreach (CloudPlaceholderSpec child in page.Children) - { - string relativePath = relativeDirectory.Length == 0 ? child.Name : Path.Combine(relativeDirectory, child.Name); - await CloudDirectoryProvenance.RetainProjectionAsync(transaction, _syncRootPath, - relativePath, child.Kind, _shutdown.Token).ConfigureAwait(false); - } + await CloudDirectoryProvenance.RetainProjectionsAsync(transaction, _syncRootPath, + page.Children.Select(child => (relativeDirectory.Length == 0 ? child.Name : Path.Combine(relativeDirectory, child.Name), child.Kind)), + _shutdown.Token).ConfigureAwait(false); string checkpointName = GetDirectoryCheckpointName(request.NormalizedPath); if (page.ContinuationToken is null) diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryBatchProvenanceTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryBatchProvenanceTests.cs new file mode 100644 index 0000000..ee11913 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryBatchProvenanceTests.cs @@ -0,0 +1,192 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task LargeCreationBatchRefreshesEachCapturedAncestorOnce(bool providerPopulation) + { + CloudPlaceholderSpec[] children = Enumerable.Range(0, 512) + .Select(index => (CloudPlaceholderSpec)CloudFilePlaceholderSpec.CreateBuilder($"child-{index:D4}.txt", $"batch-{index}", 0).Build()) + .Append(CloudDirectoryPlaceholderSpec.CreateBuilder("Folder", "batch-folder") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build()).ToArray(); + BatchDemandProvider provider = new(children); + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(provider, + populationPolicy: CloudPopulationPolicy.Full); + CloudDirectory docs = root.FileSystem.GetDirectory("Docs"); + CloudPlaceholderBatchEntryResult innerCreated = await docs.CreatePlaceholderAsync( + CloudDirectoryPlaceholderSpec.CreateBuilder("Inner", "inner") + .WithPopulationState(CloudDirectoryPopulationState.Complete).WithInSyncState(true).Build()); + CloudDirectoryMoveProof historical = await docs.PrepareMoveAsync(root.FileSystem.Root, "Archived"); + await docs.MoveToAsync(root.FileSystem.Root, "Archived", new CloudMoveOptions(historical)); + await root.FileSystem.GetDirectory("Archived").MoveToAsync(root.FileSystem.Root, "Docs"); + CloudDirectory inner = root.FileSystem.GetDirectory("Docs/Inner"); + if (providerPopulation) + { + await inner.SetPopulationStateAsync(CloudDirectoryPopulationState.Partial); + Assert.True((await inner.InspectAsync()).PlaceholderState.HasFlag(CloudPlaceholderState.Partial)); + } + + Guid deletedId = Guid.NewGuid(); + Dictionary immutable = new(); + Dictionary ancestorBindings = new(); + await using (ICloudStateTransaction seed = await root.Store.BeginTransactionAsync()) + { + await seed.Items.UpsertAsync(new(deletedId, "deleted-batch-child", "Docs\\Inner\\deleted.txt", + CloudItemKind.File, "deleted-revision", 42, true, DateTimeOffset.UtcNow)); + foreach (string name in new[] { CloudDirectoryMoveEvidence.ProofName(historical.ProofId), + CloudDirectoryMoveEvidence.MembersName(historical.ProofId), CloudDirectoryMoveEvidence.ReceiptName(historical.ProofId) }) + { + immutable.Add(name, (await seed.Checkpoints.GetAsync(name))!.Value.ToArray()); + } + + foreach (Guid id in new[] { root.DirectoryIdentity.ItemId, innerCreated.Specification.Identity.ItemId }) + { + ancestorBindings.Add(id, (await seed.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(id)))!.Value.ToArray()); + } + + await seed.CommitAsync(); + } + + ConcurrentDictionary reads = new(StringComparer.OrdinalIgnoreCase); + ConcurrentDictionary writes = new(StringComparer.Ordinal); + TaskCompletionSource committed = new(TaskCreationOptions.RunContinuationsAsynchronously); + root.Faults.SubtreeRead = path => reads.AddOrUpdate(path, 1, static (_, count) => count + 1); + root.Faults.CheckpointWritten = name => writes.AddOrUpdate(name, 1, static (_, count) => count + 1); + root.Faults.AfterCommit = () => committed.TrySetResult(); + try + { + if (providerPopulation) + { + await Task.Run(() => Directory.GetFileSystemEntries(inner.FullPath)).WaitAsync(TimeSpan.FromSeconds(30)); + using Process enumeration = Process.Start(new ProcessStartInfo + { + FileName = "cmd.exe", + Arguments = $"/c dir /b \"{inner.FullPath}\"", + UseShellExecute = false, + CreateNoWindow = true, + })!; + try + { + await enumeration.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(30)); + } + finally + { + if (!enumeration.HasExited) + { + enumeration.Kill(entireProcessTree: true); + await enumeration.WaitForExitAsync(); + } + } + + Assert.Equal(1, provider.RequestCount); + await committed.Task.WaitAsync(TimeSpan.FromSeconds(30)); + } + else + { + CloudPlaceholderBatchResult created = await inner.CreatePlaceholdersAsync(children); + Assert.True(created.IsSuccessful); + Assert.Equal(children.Length, created.SucceededCount); + Assert.All(created.Entries, entry => Assert.True(entry.DurableStatePersisted)); + } + } + finally + { + root.Faults.SubtreeRead = null; + root.Faults.CheckpointWritten = null; + root.Faults.AfterCommit = null; + } + + Assert.Equal(3, reads.Count); + Assert.All(reads.Values, count => Assert.Equal(1, count)); + Assert.Equal(1, reads["Docs"]); + Assert.Equal(1, reads["Docs\\Inner"]); + Assert.Equal(1, reads["Docs\\Inner\\Folder"]); + CloudItemSnapshot folder = await root.FileSystem.GetDirectory("Docs/Inner/Folder").InspectAsync(); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + foreach ((Guid id, byte[] binding) in ancestorBindings) + { + Assert.Equal(binding, (await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(id)))!.Value.ToArray()); + Assert.False(writes.ContainsKey(CloudDirectoryProvenance.BindingName(id))); + } + + foreach ((string name, byte[] value) in immutable) + { + Assert.Equal(value, (await verify.Checkpoints.GetAsync(name))!.Value.ToArray()); + Assert.False(writes.ContainsKey(name)); + } + + foreach ((Guid id, string path) in new[] { (root.DirectoryIdentity.ItemId, "Docs"), + (innerCreated.Specification.Identity.ItemId, "Docs\\Inner"), (children[^1].Identity.ItemId, "Docs\\Inner\\Folder") }) + { + Assert.Equal(1, writes[CloudDirectoryProvenance.MembersName(id)]); + CloudDirectoryProvenance provenance = CloudDirectoryProvenance.Decode( + (await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(id)))!.Value); + IReadOnlyList manifest = CloudDirectoryMoveEvidence.DecodeMembers( + (await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.MembersName(id)))!.Value, provenance.EvidenceId, id); + IReadOnlyList members = await verify.Items.ListSubtreeAsync(path); + Assert.Equal(members.Select(item => item.ItemId).Order(), manifest.Select(item => item.ItemId).Order()); + Assert.All(members, item => Assert.Contains(manifest, member => member.ItemId == item.ItemId && member.Kind == item.Kind && + member.Suffix == CloudDirectoryStateProjection.MapPath(item.RelativePath, path, string.Empty))); + } + + Assert.Equal(1, writes[CloudDirectoryProvenance.BindingName(children[^1].Identity.ItemId)]); + CloudDirectoryProvenance newBinding = CloudDirectoryProvenance.Decode( + (await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(children[^1].Identity.ItemId)))!.Value); + Assert.Equal(folder.LocalBinding, newBinding.Binding); + Assert.Equal(folder.PlaceholderIdentity.ToArray(), newBinding.Identity); + Assert.Equal(512, (await verify.Items.ListSubtreeAsync("Docs\\Inner")).Count(item => item.RemoteId.StartsWith("batch-", StringComparison.Ordinal) && item.Kind == CloudItemKind.File)); + CloudItemState deleted = (await verify.Items.GetByItemIdAsync(deletedId))!; + Assert.True(deleted.IsTombstone); + Assert.Equal("deleted-revision", deleted.RemoteRevision); + Assert.Equal(42, deleted.LocalFileId); + } + + [Fact] + public async Task CreationBatchOmitsOversizedLiveMembershipAndPreservesPreparation() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudDirectoryMoveProof original = await root.FileSystem.GetDirectory("Docs").PrepareMoveAsync(root.FileSystem.Root, "Moved"); + byte[] originalMembers; + string suffix = string.Join('\\', Enumerable.Repeat(new string('æ–‡', 250), 45)); + await using (ICloudStateTransaction seed = await root.Store.BeginTransactionAsync()) + { + originalMembers = (await seed.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.MembersName(original.ProofId)))!.Value.ToArray(); + await seed.Items.UpsertAsync(new(Guid.NewGuid(), "oversized-deleted", Path.Combine("Docs", suffix), + CloudItemKind.File, null, null, true, DateTimeOffset.UtcNow)); + await seed.CommitAsync(); + } + + CloudDirectoryPlaceholderSpec directory = CloudDirectoryPlaceholderSpec.CreateBuilder("Folder", "bounded-folder") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build(); + CloudPlaceholderBatchResult created = await root.FileSystem.GetDirectory("Docs").CreatePlaceholdersAsync( + [CloudFilePlaceholderSpec.CreateBuilder("new.txt", "bounded-file", 0).Build(), directory]); + Assert.True(created.IsSuccessful); + Assert.All(created.Entries, entry => Assert.True(entry.DurableStatePersisted)); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(root.DirectoryIdentity.ItemId))); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.MembersName(root.DirectoryIdentity.ItemId))); + Assert.NotNull(await verify.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(directory.Identity.ItemId))); + Assert.Equal(original.Encode(), (await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ProofName(original.ProofId)))!.Value.ToArray()); + Assert.Equal(originalMembers, (await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.MembersName(original.ProofId)))!.Value.ToArray()); + } + + private sealed class BatchDemandProvider(CloudPlaceholderSpec[] children) : ICloudDemandProvider + { + internal int RequestCount { get; private set; } + public ValueTask OpenReadAsync(CloudFileFetchRequest request, CancellationToken cancellationToken) => + throw new NotSupportedException(); + public ValueTask FetchChildrenAsync(CloudProviderFetchPlaceholdersRequest request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + RequestCount++; + return ValueTask.FromResult(new CloudProviderDirectoryPage(children, totalCount: children.Length)); + } + } +} diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs index 9f99920..595c70b 100644 --- a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveTests.cs @@ -104,7 +104,8 @@ internal async Task RestartAsync() await fileSystem.StartAsync(); } - internal static async Task OpenAsync(ICloudFileContentProvider? provider = null, bool legacyState = false) + internal static async Task OpenAsync(ICloudFileContentProvider? provider = null, bool legacyState = false, + CloudPopulationPolicy populationPolicy = CloudPopulationPolicy.Partial) { string area = Path.Combine(Path.GetTempPath(), "CfSharp-directory-move-tests", Guid.NewGuid().ToString("N")); string rootPath = Path.Combine(area, "root"); @@ -118,7 +119,7 @@ internal static async Task OpenAsync(ICloudFileContentPro CloudFileSystem fileSystem = CloudFileSystem.CreateBuilder(rootPath).WithStateStore(factory) .WithRegistration(SyncRootRegistrationOptions.CreateBuilder("CfSharp Directory Move", "1.0-test") .WithProviderId(providerId).WithSyncRootIdentity(providerId.ToByteArray()) - .WithHydrationPolicy(CloudHydrationPolicy.Progressive).WithPopulationPolicy(CloudPopulationPolicy.Partial) + .WithHydrationPolicy(CloudHydrationPolicy.Progressive).WithPopulationPolicy(populationPolicy) .WithRootMarkedInSync().Build()).WithContentProvider(provider ?? new EmptyContentProvider()).Build(); bool registered = false; try @@ -169,14 +170,16 @@ internal sealed class DirectoryMoveFaultStore(ICloudStateStore inner) : ICloudSt { internal Action? BeforeCommit { get; set; } internal Action? AfterCommit { get; set; } + internal Action? SubtreeRead { get; set; } + internal Action? CheckpointWritten { get; set; } public async ValueTask BeginTransactionAsync(CancellationToken cancellationToken = default) => new Transaction(await inner.BeginTransactionAsync(cancellationToken), this); public ValueTask DisposeAsync() => inner.DisposeAsync(); private sealed class Transaction(ICloudStateTransaction transaction, DirectoryMoveFaultStore owner) : ICloudStateTransaction { - public ICloudItemStateRepository Items => transaction.Items; - public ICloudCheckpointRepository Checkpoints => transaction.Checkpoints; + public ICloudItemStateRepository Items => owner.SubtreeRead is null ? transaction.Items : new ObservedItems(transaction.Items, owner); + public ICloudCheckpointRepository Checkpoints => owner.CheckpointWritten is null ? transaction.Checkpoints : new ObservedCheckpoints(transaction.Checkpoints, owner); public ICloudOperationJournal Operations => transaction.Operations; public ICloudConflictRepository Conflicts => transaction.Conflicts; public ICloudRemoteBatchRepository RemoteBatches => transaction.RemoteBatches; @@ -191,4 +194,37 @@ public async ValueTask CommitAsync(CancellationToken cancellationToken = default public ValueTask RollbackAsync(CancellationToken cancellationToken = default) => transaction.RollbackAsync(cancellationToken); public ValueTask DisposeAsync() => transaction.DisposeAsync(); } + + private sealed class ObservedItems(ICloudItemStateRepository items, DirectoryMoveFaultStore owner) : ICloudItemStateRepository + { + public ValueTask GetByItemIdAsync(Guid itemId, CancellationToken cancellationToken = default) => + items.GetByItemIdAsync(itemId, cancellationToken); + public ValueTask GetByRemoteIdAsync(string remoteId, CancellationToken cancellationToken = default) => + items.GetByRemoteIdAsync(remoteId, cancellationToken); + public ValueTask GetByRelativePathAsync(string relativePath, CancellationToken cancellationToken = default) => + items.GetByRelativePathAsync(relativePath, cancellationToken); + public ValueTask> ListSubtreeAsync(string relativePath, CancellationToken cancellationToken = default) + { + owner.SubtreeRead?.Invoke(relativePath); + return items.ListSubtreeAsync(relativePath, cancellationToken); + } + + public ValueTask UpsertAsync(CloudItemState item, CancellationToken cancellationToken = default) => items.UpsertAsync(item, cancellationToken); + public ValueTask RemoveAsync(Guid itemId, CancellationToken cancellationToken = default) => items.RemoveAsync(itemId, cancellationToken); + } + + private sealed class ObservedCheckpoints(ICloudCheckpointRepository checkpoints, DirectoryMoveFaultStore owner) : ICloudCheckpointRepository + { + public ValueTask GetAsync(string name, CancellationToken cancellationToken = default) => + checkpoints.GetAsync(name, cancellationToken); + public ValueTask> ListAsync(string namePrefix, CancellationToken cancellationToken = default) => + checkpoints.ListAsync(namePrefix, cancellationToken); + public ValueTask UpsertAsync(CloudStateCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + owner.CheckpointWritten?.Invoke(checkpoint.Name); + return checkpoints.UpsertAsync(checkpoint, cancellationToken); + } + + public ValueTask RemoveAsync(string name, CancellationToken cancellationToken = default) => checkpoints.RemoveAsync(name, cancellationToken); + } } From c0aa286f015f71fb3571ac7a80829afd71e3a3b1 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:12:26 +0800 Subject: [PATCH 34/40] fix(file-system): use native parent spelling for directory moves --- docs/placeholders.md | 3 + ...loudFileSystem.DirectoryMovePreparation.cs | 19 +++ .../CloudFileSystem.NamespaceOperations.cs | 27 +++- .../Items/CloudDirectory.MovePreparation.cs | 2 + .../CloudDirectoryMoveParentCasingTests.cs | 125 ++++++++++++++++++ 5 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveParentCasingTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index 389069c..e5897c5 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -13,6 +13,9 @@ Each preparation has its own ID and survives restart independently. Do not subst observed at a renamed target for missing historical evidence. Ordinary directories may use normal `MoveToAsync`, but cannot prepare external recovery without a managed placeholder identity and complete native IDs. Metadata capture reads no content and holds no continuing lock after returning. +The proof records the validated parent's actual native spelling and the requested child name. +Parent references may differ in casing; the final child name remains exact for case-only renames +and must match during both facade and feed recovery. Directory creation, conversion, identity replacement, remote creation, and provider population retain native provenance in the same transaction as their official item rows. Adding children diff --git a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs index 270cb97..d73bdf8 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.DirectoryMovePreparation.cs @@ -32,9 +32,28 @@ internal async ValueTask PrepareDirectoryMoveAsync(Clou throw new DirectoryNotFoundException("The destination parent does not exist."); } + target = ResolveDirectoryMoveTarget(destination, validatedName); return await PrepareDirectoryMoveCoreAsync(source, target, operation, cancellationToken).ConfigureAwait(false); } + private CloudDirectory ResolveDirectoryMoveTarget(CloudDirectory destination, string name) => + (CloudDirectory)CreateItemReference(Path.GetRelativePath(SyncRootPath, + Path.Combine(ReadDirectoryMovePath(destination), name)), CloudItemKind.Directory); + + private static string ReadDirectoryMovePath(CloudDirectory directory) + { + using SafeFileHandle guard = WindowsFileMetadata.Open(directory.FullPath, preventDelete: true); + string actualPath = WindowsFileMetadata.ReadFinalPath(guard.DangerousGetHandle()); + if (!string.Equals(actualPath, directory.FullPath, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("The native directory is outside the requested namespace path."); + } + + // References may vary in parent casing. Preserve the requested child name separately: + // normalizing the whole target would erase an intended case-only leaf rename. + return actualPath; + } + private async ValueTask PrepareDirectoryMoveCoreAsync(CloudDirectory source, CloudDirectory target, CloudFileSystemOperationLease operation, CancellationToken cancellationToken) { diff --git a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs index ce90c89..ad2b4ce 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs @@ -39,7 +39,8 @@ internal async ValueTask MoveAsync( CloudDirectoryMoveProof? directoryProof = options.DirectoryMoveProof; if (directoryProof is not null && (item.Kind != CloudItemKind.Directory || !CloudDirectoryStateProjection.SamePath(item.RelativePath, directoryProof.SourceRelativePath) || - movedItem.RelativePath != directoryProof.DestinationRelativePath)) + !CloudDirectoryStateProjection.SamePath(movedItem.RelativePath, directoryProof.DestinationRelativePath) || + validatedName != Path.GetFileName(directoryProof.DestinationRelativePath))) { throw new ArgumentException("The directory proof must identify this source and exact intended destination.", nameof(options)); } @@ -74,6 +75,17 @@ internal async ValueTask MoveAsync( $"The destination directory does not exist: '{destination.FullPath}'."); } + if (item is CloudDirectory) + { + // Resolve under the admitted lease, without entering a public reference API that + // would reject Stopping. Its path scopes already cover the case-insensitive alias. + movedItem = ResolveDirectoryMoveTarget(destination, validatedName); + if (directoryProof is not null && movedItem.RelativePath != directoryProof.DestinationRelativePath) + { + throw new ArgumentException("The directory proof must identify this source and exact intended destination.", nameof(options)); + } + } + if (item is CloudDirectory directory && (directoryProof is not null || !sourceState.Exists)) { directoryProof ??= await CloudDirectoryMoveEvidence.ReadIntentAsync(operation.StateStore, @@ -160,7 +172,16 @@ await PersistMovedStateAsync( statePlan.SourceEntries.Count); } - if (string.Equals(item.FullPath, movedItem.FullPath, StringComparison.Ordinal)) + string nativeSourcePath = item.FullPath; + if (item is CloudDirectory samePathDirectory && + string.Equals(item.FullPath, movedItem.FullPath, StringComparison.OrdinalIgnoreCase)) + { + // A case-insensitive source reference alone cannot distinguish a no-op from a + // case-only rename. Use the observed spelling, including for the native rename. + nativeSourcePath = ReadDirectoryMovePath(samePathDirectory); + } + + if (string.Equals(nativeSourcePath, movedItem.FullPath, StringComparison.Ordinal)) { CloudItemSnapshot unchangedSnapshot = await InspectCoreAsync( movedItem, @@ -197,7 +218,7 @@ await PersistMovedStateAsync( } else { - Directory.Move(item.FullPath, movedItem.FullPath); + Directory.Move(nativeSourcePath, movedItem.FullPath); } } catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) diff --git a/src/CfSharp/Items/CloudDirectory.MovePreparation.cs b/src/CfSharp/Items/CloudDirectory.MovePreparation.cs index 8338cb5..cbb41dd 100644 --- a/src/CfSharp/Items/CloudDirectory.MovePreparation.cs +++ b/src/CfSharp/Items/CloudDirectory.MovePreparation.cs @@ -16,6 +16,8 @@ public sealed partial class CloudDirectory /// the returned value confers no continuing protection. Concurrent operations are coordinated /// within this file system. Cross-process identity and descendant changes still require recovery /// validation. Ordinary directories retain their existing normal move API. + /// The destination parent uses its observed native spelling; the requested child name remains + /// exact, including case-only renames. Parent reference casing does not change the proof target. /// /// The destination belongs to another file system or the name is invalid. /// The root, destination, or durable/native identity is unsuitable. diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveParentCasingTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveParentCasingTests.cs new file mode 100644 index 0000000..d38eddc --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveParentCasingTests.cs @@ -0,0 +1,125 @@ +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData("automatic")] + [InlineData("prepared")] + [InlineData("feed")] + [InlineData("retry")] + public async Task DirectoryMoveUsesNativeParentSpellingAndPreservesRequestedChildCase(string mode) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + Directory.CreateDirectory(Path.Combine(root.RootPath, "Parent")); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectory destination = root.FileSystem.GetDirectory("parent"); + CloudDirectoryMoveProof? proof = null; + CloudDirectoryMoveReconciliationResult recovery; + if (mode == "feed") + { + proof = await source.PrepareMoveAsync(destination, "Moved"); + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Parent", "Moved")); + await ProcessRenameAsync(root, "Docs", "Parent\\Moved"); + recovery = await source.ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.AlreadyProjected, recovery.Outcome); + } + else + { + if (mode == "prepared") + { + proof = await source.PrepareMoveAsync(destination, "Moved"); + } + + if (mode == "retry") + { + IOException storageError = new("Projection commit fault."); + int commits = 0; + root.Faults.BeforeCommit = () => + { + if (++commits == 2) + { + throw storageError; + } + }; + try + { + CloudItemCoordinationException failed = await Assert.ThrowsAsync( + () => source.MoveToAsync(destination, "Moved").AsTask()); + Assert.Same(storageError, failed.InnerException); + Assert.True(failed.DirectoryReconciliation!.NativeMoveObserved); + Assert.False(failed.DirectoryReconciliation.DurableProjectionCommitted); + } + finally + { + root.Faults.BeforeCommit = null; + } + + destination = root.FileSystem.GetDirectory("PARENT"); + } + + CloudItemMoveResult result = await source.MoveToAsync(destination, "Moved", + proof is null ? CloudMoveOptions.Default : new CloudMoveOptions(proof)); + Assert.Equal("Parent\\Moved", result.Item.RelativePath); + recovery = result.DirectoryReconciliation!; + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, recovery.Outcome); + } + + Assert.True(recovery.DurableProjectionCommitted); + Assert.False(recovery.RequiresFullRescan); + Assert.False(Directory.Exists(source.FullPath)); + Assert.True(Directory.Exists(Path.Combine(root.RootPath, "Parent", "Moved"))); + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + proof ??= CloudDirectoryMoveProof.Decode((await verify.Checkpoints.GetAsync( + CloudDirectoryMoveEvidence.IntentName("Docs", "Parent\\Moved")))!.Value.Span); + Assert.Equal("Parent\\Moved", proof.DestinationRelativePath); + Assert.Equal(proof.Encode(), (await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId)))!.Value.ToArray()); + Assert.Equal("Parent\\Moved", (await verify.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!.RelativePath); + Assert.NotNull(await verify.Items.GetByRelativePathAsync("Parent\\Moved\\child.txt")); + } + + // Canonicalizing the parent must not canonicalize away an intended case-only leaf rename. + CloudItemMoveResult renamed = await root.FileSystem.GetDirectory("Parent/Moved") + .MoveToAsync(root.FileSystem.GetDirectory("PARENT"), "moved"); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, renamed.DirectoryReconciliation!.Outcome); + Assert.Equal("Parent\\moved", renamed.Item.RelativePath); + await using (ICloudStateTransaction after = await root.Store.BeginTransactionAsync()) + { + Assert.Equal("Parent\\moved", (await after.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!.RelativePath); + Assert.Equal("Parent\\moved\\child.txt", (await after.Items.GetByRemoteIdAsync("child"))!.RelativePath); + } + + CloudItemMoveResult unchanged = await root.FileSystem.GetDirectory("parent/moved") + .MoveToAsync(root.FileSystem.GetDirectory("PARENT"), "moved"); + Assert.Equal(0, unchanged.DurableStateEntriesUpdated); + Assert.Null(unchanged.DirectoryReconciliation); + CloudItemMoveResult fromAlias = await root.FileSystem.GetDirectory("PARENT/MOVED") + .MoveToAsync(destination, "MOVED"); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, fromAlias.DirectoryReconciliation!.Outcome); + Assert.Equal("Parent\\MOVED", fromAlias.Item.RelativePath); + } + + [Fact] + public async Task CanonicalParentDoesNotPermitAnotherFinalNameSpelling() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + Directory.CreateDirectory(Path.Combine(root.RootPath, "Parent")); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + CloudDirectory destination = root.FileSystem.GetDirectory("parent"); + CloudDirectoryMoveProof proof = await source.PrepareMoveAsync(destination, "moved"); + await Assert.ThrowsAsync(() => source.MoveToAsync(destination, "MOVED", new CloudMoveOptions(proof)).AsTask()); + Assert.True(Directory.Exists(source.FullPath)); + Directory.Move(source.FullPath, Path.Combine(root.RootPath, "Parent", "MOVED")); + CloudDirectoryMoveReconciliationResult result = await source.ReconcileMoveAsync(proof); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Conflict, result.Outcome); + Assert.True(result.RequiresFullRescan); + Assert.False(result.DurableProjectionCommitted); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal("Docs", (await verify.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!.RelativePath); + Assert.NotNull(await verify.Items.GetByRelativePathAsync("Docs\\child.txt")); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + } +} From 8c90987d2c2448de5e24ee111e240a48904cf307 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:27:12 +0800 Subject: [PATCH 35/40] fix(persistence): refresh ancestor membership after directory moves --- docs/placeholders.md | 8 +- .../CloudDirectory.MoveReconciliation.cs | 4 +- .../Persistence/CloudDirectoryProvenance.cs | 37 +++- ...oudDirectoryMoveAncestorProvenanceTests.cs | 208 ++++++++++++++++++ 4 files changed, 243 insertions(+), 14 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveAncestorProvenanceTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index e5897c5..30362bd 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -31,8 +31,12 @@ After the external move, call `await originalDirectory.ReconcileMoveAsync(proof) source reference. Recovery never issues another native move. It validates the original stored preparation, complete native root/directory binding and opaque identity, actual namespace spelling, and known current membership. Source and destination paths and an immutable completion receipt -commit together. Current revisions, local IDs, tombstones, and pending journal IDs/sequences/payloads -are retained; a root already at the target is still a proven member. Unrelated target rows remain +commit together. The same transaction refreshes retained live membership of source and destination +ancestors once each, including the common parent of a renamed child. This allows a later external +parent move to recover from its current membership; ancestor bindings and immutable historical +preparations, manifests, and receipts remain unchanged. Current revisions, local IDs, tombstones, +and pending journal IDs/sequences/payloads are retained; a root already at the target is still a +proven member. Unrelated target rows remain untouched and cause a conflict. A historical receipt cannot regress later state. Inspect `Outcome`, `Stage`, `NativeMoveObserved`, `DurableProjectionCommitted`, and diff --git a/src/CfSharp/Items/CloudDirectory.MoveReconciliation.cs b/src/CfSharp/Items/CloudDirectory.MoveReconciliation.cs index 1a5dd4b..c73af90 100644 --- a/src/CfSharp/Items/CloudDirectory.MoveReconciliation.cs +++ b/src/CfSharp/Items/CloudDirectory.MoveReconciliation.cs @@ -11,7 +11,9 @@ public sealed partial class CloudDirectory /// 1709 and complete native IDs. This method never moves a directory, rewrites native identity, /// hydrates content, clears dirty state, or acknowledges journal entries. It authenticates the /// proof in the same durable store, protects parent paths and the target with metadata guards, - /// and validates native observations around one atomic path/receipt transaction. Such guards + /// and validates native observations around one atomic path/receipt transaction. The transaction + /// also refreshes retained live membership of source and destination ancestors once each; + /// their bindings and historical preparations and receipts remain unchanged. Such guards /// do not freeze content, identity, or descendant changes across processes. Retry a pending /// projection with the same proof; conflicts require reconciliation, not a guessed replacement. /// The result owns no protection lifetime. Concurrent calls are coordinated within this owner. diff --git a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs index 3c06f91..daa288a 100644 --- a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs +++ b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs @@ -92,11 +92,7 @@ internal static async ValueTask RetainProjectionsAsync(ICloudStateTransaction tr captured.Add(relativePath, provenance); } - string? parent = Path.GetDirectoryName(relativePath); - while (!string.IsNullOrEmpty(parent) && ancestors.Add(parent)) - { - parent = Path.GetDirectoryName(parent); - } + AddAncestors(ancestors, relativePath); } // All item rows must already be written. Capture each new native binding first, then @@ -107,15 +103,26 @@ internal static async ValueTask RetainProjectionsAsync(ICloudStateTransaction tr await RetainMetadataAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); } + ancestors.ExceptWith(captured.Keys); + await RefreshAncestorsAsync(transaction, ancestors, cancellationToken).ConfigureAwait(false); + } + + private static void AddAncestors(HashSet ancestors, string relativePath) + { + string? parent = Path.GetDirectoryName(relativePath); + while (!string.IsNullOrEmpty(parent) && ancestors.Add(parent)) + { + parent = Path.GetDirectoryName(parent); + } + } + + private static async ValueTask RefreshAncestorsAsync(ICloudStateTransaction transaction, + IEnumerable ancestors, CancellationToken cancellationToken) + { // Refresh existing bindings without traversing native subtrees or manufacturing new - // historical evidence. Shared ancestors of siblings use the same final row snapshot. + // historical evidence. Shared ancestors use the same final row snapshot once. foreach (string parent in ancestors) { - if (captured.ContainsKey(parent)) - { - continue; - } - CloudItemState? directory = await transaction.Items.GetByRelativePathAsync(parent, cancellationToken).ConfigureAwait(false); if (directory is { Kind: CloudItemKind.Directory, IsTombstone: false }) { @@ -202,6 +209,14 @@ internal static async ValueTask ProjectPathsAsync(ICloudStateTransaction transac await RetainMetadataAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); } + + // A child move changes membership outside the moved subtree as well. Refresh both + // ancestor chains from the final official rows in the projection/receipt transaction, + // retaining their native bindings and every immutable preparation and receipt. + HashSet ancestors = new(StringComparer.OrdinalIgnoreCase); + AddAncestors(ancestors, proof.SourceRelativePath); + AddAncestors(ancestors, proof.DestinationRelativePath); + await RefreshAncestorsAsync(transaction, ancestors, cancellationToken).ConfigureAwait(false); } private static async ValueTask RetainMetadataAsync(ICloudStateTransaction transaction, CloudDirectoryProvenance provenance, diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveAncestorProvenanceTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveAncestorProvenanceTests.cs new file mode 100644 index 0000000..b2e3523 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveAncestorProvenanceTests.cs @@ -0,0 +1,208 @@ +using System.Collections.Concurrent; +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ChildMoveRefreshesAncestorsAndAllowsLaterParentFeedRecovery(bool crossParent, bool childViaFeed) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + AncestorMoveScenario scenario = await PrepareAncestorMoveScenarioAsync(root, crossParent); + ConcurrentDictionary reads = new(StringComparer.OrdinalIgnoreCase); + ConcurrentDictionary writes = new(StringComparer.Ordinal); + root.Faults.SubtreeRead = path => reads.AddOrUpdate(path, 1, static (_, count) => count + 1); + root.Faults.CheckpointWritten = name => writes.AddOrUpdate(name, 1, static (_, count) => count + 1); + try + { + if (childViaFeed) + { + Directory.Move(Path.Combine(root.RootPath, scenario.Source), Path.Combine(root.RootPath, scenario.Target)); + await ProcessRenameAsync(root, scenario.Source, scenario.Target); + } + else + { + CloudItemMoveResult moved = await root.FileSystem.GetDirectory(scenario.Source).MoveToAsync( + root.FileSystem.GetDirectory(Path.GetDirectoryName(scenario.Target)!), Path.GetFileName(scenario.Target)); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, moved.DirectoryReconciliation!.Outcome); + } + } + finally + { + root.Faults.SubtreeRead = null; + root.Faults.CheckpointWritten = null; + } + + Dictionary membershipMatches = new(); + Dictionary afterChildBindings = new(); + await using (ICloudStateTransaction afterChild = await root.Store.BeginTransactionAsync()) + { + foreach ((Guid id, CloudStateCheckpoint[] image) in scenario.Ancestors) + { + CloudStateCheckpoint binding = (await afterChild.Checkpoints.GetAsync(image[0].Name))!; + afterChildBindings.Add(binding.Name, binding); + CloudDirectoryProvenance provenance = CloudDirectoryProvenance.Decode(binding.Value); + IReadOnlyList members = CloudDirectoryMoveEvidence.DecodeMembers( + (await afterChild.Checkpoints.GetAsync(image[1].Name))!.Value, provenance.EvidenceId, id); + IReadOnlyList rows = await afterChild.Items.ListSubtreeAsync(provenance.RelativePath); + membershipMatches.Add(id, members.Select(member => (member.ItemId, member.Kind, member.Suffix)).OrderBy(member => member.ItemId) + .SequenceEqual(rows.Select(item => (item.ItemId, item.Kind, + CloudDirectoryStateProjection.MapPath(item.RelativePath, provenance.RelativePath, string.Empty))).OrderBy(item => item.ItemId))); + } + } + + // Promote the parent's latest mutable membership without supplying a new caller proof. + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + await ProcessRenameAsync(root, "Docs", "Moved"); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal("Moved", (await verify.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!.RelativePath); + Assert.False(LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + string projectedInner = CloudDirectoryStateProjection.MapPath(scenario.Target, "Docs", "Moved"); + Assert.Equal(projectedInner, (await verify.Items.GetByItemIdAsync(scenario.InnerId))!.RelativePath); + CloudItemState leaf = (await verify.Items.GetByItemIdAsync(scenario.Pending.ItemId!.Value))!; + Assert.Equal(Path.Combine(projectedInner, "leaf.txt"), leaf.RelativePath); + Assert.Equal("current-leaf", leaf.RemoteRevision); + Assert.Equal(33, leaf.LocalFileId); + CloudItemState deleted = (await verify.Items.GetByItemIdAsync(scenario.DeletedId))!; + Assert.Equal(Path.Combine(projectedInner, "deleted.txt"), deleted.RelativePath); + Assert.True(deleted.IsTombstone); + Assert.Equal("deleted-revision", deleted.RemoteRevision); + Assert.Equal(42, deleted.LocalFileId); + CloudOperationJournalEntry pending = (await verify.Operations.GetAsync(scenario.Pending.OperationId))!; + Assert.Equal(scenario.Pending.Sequence, pending.Sequence); + Assert.Equal(scenario.Pending.Payload.ToArray(), pending.Payload.ToArray()); + Assert.Equal(scenario.Pending.CreatedAt, pending.CreatedAt); + Assert.All(membershipMatches.Values, matches => Assert.True(matches)); + foreach ((Guid id, CloudStateCheckpoint[] image) in scenario.Ancestors) + { + CloudDirectoryProvenance provenance = CloudDirectoryProvenance.Decode(image[0].Value); + Assert.Equal(1, reads.GetValueOrDefault(provenance.RelativePath)); + Assert.Equal(1, writes.GetValueOrDefault(CloudDirectoryProvenance.MembersName(id))); + CloudStateCheckpoint binding = afterChildBindings[image[0].Name]; + Assert.Equal(image[0].Value.ToArray(), binding.Value.ToArray()); + Assert.Equal(image[0].UpdatedAt, binding.UpdatedAt); + } + + await AssertCheckpointImageAsync(verify, scenario.History); + Assert.Equal(childViaFeed ? 2 : 1, (await verify.Operations.ListAsync(100)).Count(item => item.Kind == CloudStateOperationKind.Move)); + } + + [Fact] + public async Task AncestorMembershipWriteFailureRollsBackProjectionAndOriginalRetryRecovers() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + AncestorMoveScenario scenario = await PrepareAncestorMoveScenarioAsync(root, crossParent: true); + CloudDirectory source = root.FileSystem.GetDirectory(scenario.Source); + CloudDirectory destination = root.FileSystem.GetDirectory(Path.GetDirectoryName(scenario.Target)!); + IOException storageError = new("Ancestor membership write fault."); + root.Faults.CheckpointWritten = name => + { + if (name == CloudDirectoryProvenance.MembersName(root.DirectoryIdentity.ItemId)) + { + throw storageError; + } + }; + try + { + CloudItemCoordinationException failed = await Assert.ThrowsAsync( + () => source.MoveToAsync(destination, "Renamed").AsTask()); + Assert.Same(storageError, failed.InnerException); + Assert.True(failed.DirectoryReconciliation!.NativeMoveObserved); + Assert.False(failed.DirectoryReconciliation.DurableProjectionCommitted); + } + finally + { + root.Faults.CheckpointWritten = null; + } + + Assert.False(Directory.Exists(source.FullPath)); + Assert.True(Directory.Exists(Path.Combine(root.RootPath, scenario.Target))); + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + Assert.Equal(scenario.Source, (await verify.Items.GetByItemIdAsync(scenario.InnerId))!.RelativePath); + Assert.Equal(Path.Combine(scenario.Source, "leaf.txt"), (await verify.Items.GetByItemIdAsync(scenario.Pending.ItemId!.Value))!.RelativePath); + foreach (CloudStateCheckpoint[] image in scenario.Ancestors.Values) + { + await AssertCheckpointImageAsync(verify, image); + } + + CloudDirectoryMoveProof prepared = CloudDirectoryMoveProof.Decode((await verify.Checkpoints.GetAsync( + CloudDirectoryMoveEvidence.IntentName(scenario.Source, scenario.Target)))!.Value.Span); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(prepared.ProofId))); + await AssertCheckpointImageAsync(verify, scenario.History); + } + + CloudItemMoveResult recovered = await source.MoveToAsync(destination, "Renamed"); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, recovered.DirectoryReconciliation!.Outcome); + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + await ProcessRenameAsync(root, "Docs", "Moved"); + await using ICloudStateTransaction after = await root.Store.BeginTransactionAsync(); + Assert.Equal("Moved", (await after.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!.RelativePath); + Assert.False(LocalChangeCheckpoint.Decode((await after.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + await AssertCheckpointImageAsync(after, scenario.History); + } + + private static async Task PrepareAncestorMoveScenarioAsync(DirectoryMoveTestRoot root, bool crossParent) + { + CloudDirectory docs = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryPlaceholderSpec left = CloudDirectoryPlaceholderSpec.CreateBuilder("Left", "left") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build(); + CloudDirectoryPlaceholderSpec right = CloudDirectoryPlaceholderSpec.CreateBuilder("Right", "right") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build(); + await docs.CreatePlaceholdersAsync(crossParent ? [left, right] : [left]); + CloudDirectoryPlaceholderSpec inner = CloudDirectoryPlaceholderSpec.CreateBuilder("Inner", "inner") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build(); + await root.FileSystem.GetDirectory("Docs/Left").CreatePlaceholderAsync(inner); + CloudFilePlaceholderSpec leaf = CloudFilePlaceholderSpec.CreateBuilder("leaf.txt", "nested-leaf", 0).Build(); + string sourcePath = "Docs\\Left\\Inner"; + await root.FileSystem.GetDirectory(sourcePath).CreatePlaceholderAsync(leaf); + Guid deletedId = Guid.NewGuid(); + CloudOperationJournalEntry pending; + await using (ICloudStateTransaction seed = await root.Store.BeginTransactionAsync()) + { + await seed.Items.UpsertAsync(new(deletedId, "nested-deleted", Path.Combine(sourcePath, "deleted.txt"), + CloudItemKind.File, "deleted-revision", 42, true, DateTimeOffset.UtcNow)); + CloudItemState current = (await seed.Items.GetByItemIdAsync(leaf.Identity.ItemId))!; + await seed.Items.UpsertAsync(new(current.ItemId, current.RemoteId, current.RelativePath, + current.Kind, "current-leaf", 33, false, DateTimeOffset.UtcNow)); + pending = await seed.Operations.EnqueueAsync(new(Guid.NewGuid(), CloudStateOperationKind.ContentUpdate, current.ItemId, + new LocalChangePayload(current.RelativePath, null, false, DateTimeOffset.UtcNow).Encode(), DateTimeOffset.UtcNow)); + await seed.CommitAsync(); + } + + CloudDirectoryMoveProof historical = await docs.PrepareMoveAsync(root.FileSystem.Root, "Archived"); + await docs.MoveToAsync(root.FileSystem.Root, "Archived", new CloudMoveOptions(historical)); + await root.FileSystem.GetDirectory("Archived").MoveToAsync(root.FileSystem.Root, "Docs"); + Dictionary ancestors = new(); + await using ICloudStateTransaction read = await root.Store.BeginTransactionAsync(); + foreach (Guid id in new[] { root.DirectoryIdentity.ItemId, left.Identity.ItemId }.Concat(crossParent ? [right.Identity.ItemId] : Array.Empty())) + { + ancestors.Add(id, [(await read.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(id)))!, + (await read.Checkpoints.GetAsync(CloudDirectoryProvenance.MembersName(id)))!]); + } + + CloudStateCheckpoint[] history = (await read.Checkpoints.ListAsync($"cfsharp/namespace/preparations/{historical.ProofId:N}")).ToArray(); + Assert.Equal(3, history.Length); + return new(sourcePath, crossParent ? "Docs\\Right\\Renamed" : "Docs\\Left\\Renamed", inner.Identity.ItemId, deletedId, + pending, ancestors, history); + } + + private static async Task AssertCheckpointImageAsync(ICloudStateTransaction transaction, IEnumerable image) + { + foreach (CloudStateCheckpoint saved in image) + { + CloudStateCheckpoint current = (await transaction.Checkpoints.GetAsync(saved.Name))!; + Assert.Equal(saved.Value.ToArray(), current.Value.ToArray()); + Assert.Equal(saved.UpdatedAt, current.UpdatedAt); + } + } + + private sealed record AncestorMoveScenario(string Source, string Target, Guid InnerId, Guid DeletedId, + CloudOperationJournalEntry Pending, IReadOnlyDictionary Ancestors, CloudStateCheckpoint[] History); +} From 57392ddd237e1c850c14ad761e6696bf89448e98 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:32:36 +0800 Subject: [PATCH 36/40] fix(persistence): refresh ancestor membership after file moves --- docs/placeholders.md | 9 + .../CloudFileSystem.NamespaceOperations.cs | 2 + .../Items/CloudItem.NamespaceOperations.cs | 5 + .../LocalChanges/CloudLocalChangeFeed.cs | 10 + .../Persistence/CloudDirectoryProvenance.cs | 17 +- .../CloudFileMoveProvenanceTests.cs | 273 ++++++++++++++++++ 6 files changed, 311 insertions(+), 5 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudFileMoveProvenanceTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index 30362bd..3cfcd87 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -27,6 +27,15 @@ prepared proofs remain unchanged. Removing an identity or reverting a directory provenance. Storage without complete IDs retains existing creation behavior but cannot establish this recovery evidence; legacy directories can explicitly prepare while still at their source. +Known file moves and renames through `MoveToAsync` or the local-change feed update official paths +and refresh retained live membership of both source and destination ancestors in one transaction. +Shared ancestors are refreshed once, including for case-only renames and files moved into or out +of a directory. Existing ancestor native bindings and immutable preparations, manifests and receipts +remain unchanged, and pending journal operations are not rewritten or acknowledged. If file +projection fails after the native move, retry the original source/destination, including after +restart; the facade does not move an absent source again. Feed storage failures still retain their +original unresolved observation and rescan fence until application reconciliation completes. + After the external move, call `await originalDirectory.ReconcileMoveAsync(proof)` on the original source reference. Recovery never issues another native move. It validates the original stored preparation, complete native root/directory binding and opaque identity, actual namespace spelling, diff --git a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs index ad2b4ce..495c29c 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs @@ -433,6 +433,8 @@ private static async ValueTask PersistMovedStateAsync( .ConfigureAwait(false); await CloudDirectoryStateProjection.ProjectAsync(transaction, sourceEntries, sourceRelativePath, destinationRelativePath, DateTimeOffset.UtcNow, cancellationToken).ConfigureAwait(false); + await CloudDirectoryProvenance.RefreshMoveAncestorsAsync(transaction, sourceRelativePath, + destinationRelativePath, cancellationToken).ConfigureAwait(false); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); } diff --git a/src/CfSharp/Items/CloudItem.NamespaceOperations.cs b/src/CfSharp/Items/CloudItem.NamespaceOperations.cs index d473423..b69e3b8 100644 --- a/src/CfSharp/Items/CloudItem.NamespaceOperations.cs +++ b/src/CfSharp/Items/CloudItem.NamespaceOperations.cs @@ -22,6 +22,11 @@ public abstract partial class CloudItem /// durable recovery. Concurrent owner disposal waits for that work; new public operations /// are rejected after stopping begins. Cancellation cannot interrupt durable recovery after /// the synchronous native move succeeds. + /// Known file paths and retained live membership of source and destination ancestors commit + /// together, with each ancestor refreshed once. Their native bindings, immutable directory + /// preparations and receipts, and existing pending journal operations remain unchanged. + /// A failed file projection can be retried on the original source reference and destination, + /// including after restart, without issuing another move when the source is already absent. /// /// /// The destination belongs to another file system, the name is invalid, or replacement was diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs index 2bb2051..605c4b2 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs @@ -18,6 +18,11 @@ namespace CfSharp; /// perform a full reconciliation before acknowledging that condition; periodic reconciliation is /// intentionally outside CfSharp. /// +/// +/// Known file rename paths, retained source/destination ancestor membership, and the new journal +/// observation commit in one transaction. Ancestor native bindings, immutable directory recovery +/// evidence and existing pending operations remain unchanged. Shared ancestors are refreshed once. +/// /// public sealed partial class CloudLocalChangeFeed : IDisposable, IAsyncDisposable { @@ -866,6 +871,11 @@ await UpsertCheckpointAsync( isTombstone: kind == CloudLocalChangeKind.Delete, observedAt); await transaction.Items.UpsertAsync(updated, cancellationToken).ConfigureAwait(false); + if (kind == CloudLocalChangeKind.Move && updated.Kind == CloudItemKind.File && previousPath is not null) + { + await CloudDirectoryProvenance.RefreshMoveAncestorsAsync(transaction, previousPath.Value.RelativePath, + path.RelativePath, cancellationToken).ConfigureAwait(false); + } } LocalChangePayload payload = new( diff --git a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs index daa288a..bcc36bf 100644 --- a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs +++ b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs @@ -210,12 +210,19 @@ internal static async ValueTask ProjectPathsAsync(ICloudStateTransaction transac await RetainMetadataAsync(transaction, provenance, cancellationToken).ConfigureAwait(false); } - // A child move changes membership outside the moved subtree as well. Refresh both - // ancestor chains from the final official rows in the projection/receipt transaction, - // retaining their native bindings and every immutable preparation and receipt. + await RefreshMoveAncestorsAsync(transaction, proof.SourceRelativePath, proof.DestinationRelativePath, + cancellationToken).ConfigureAwait(false); + } + + internal static async ValueTask RefreshMoveAncestorsAsync(ICloudStateTransaction transaction, + string sourceRelativePath, string destinationRelativePath, CancellationToken cancellationToken) + { + // A namespace move also changes membership outside the moved item/subtree. Refresh + // both ancestor chains from final official rows in the caller's projection transaction, + // retaining native bindings and every immutable preparation, manifest and receipt. HashSet ancestors = new(StringComparer.OrdinalIgnoreCase); - AddAncestors(ancestors, proof.SourceRelativePath); - AddAncestors(ancestors, proof.DestinationRelativePath); + AddAncestors(ancestors, sourceRelativePath); + AddAncestors(ancestors, destinationRelativePath); await RefreshAncestorsAsync(transaction, ancestors, cancellationToken).ConfigureAwait(false); } diff --git a/tests/CfSharp.IntegrationTests/CloudFileMoveProvenanceTests.cs b/tests/CfSharp.IntegrationTests/CloudFileMoveProvenanceTests.cs new file mode 100644 index 0000000..5b554e0 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudFileMoveProvenanceTests.cs @@ -0,0 +1,273 @@ +using System.Collections.Concurrent; +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData("rename", false)] + [InlineData("rename", true)] + [InlineData("case", false)] + [InlineData("case", true)] + [InlineData("cross", false)] + [InlineData("cross", true)] + [InlineData("out", false)] + [InlineData("out", true)] + [InlineData("in", false)] + [InlineData("in", true)] + public async Task FileMovesRefreshLiveAncestorsAndKeepParentFeedRecovery(string shape, bool viaFeed) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + FileMoveMembershipScenario scenario = await PrepareFileMoveMembershipAsync(root, shape); + ConcurrentDictionary reads = new(StringComparer.OrdinalIgnoreCase); + ConcurrentDictionary writes = new(StringComparer.Ordinal); + root.Faults.SubtreeRead = path => reads.AddOrUpdate(path, 1, static (_, count) => count + 1); + root.Faults.CheckpointWritten = name => writes.AddOrUpdate(name, 1, static (_, count) => count + 1); + try + { + if (viaFeed) + { + File.Move(Path.Combine(root.RootPath, scenario.Source), Path.Combine(root.RootPath, scenario.Target)); + await ProcessRenameAsync(root, scenario.Source, scenario.Target); + } + else + { + CloudItemMoveResult result = await root.FileSystem.GetFile(scenario.Source).MoveToAsync( + root.FileSystem.GetDirectory(Path.GetDirectoryName(scenario.Target)!), Path.GetFileName(scenario.Target)); + Assert.Equal(1, result.DurableStateEntriesUpdated); + Assert.Null(result.DirectoryReconciliation); + } + } + finally + { + root.Faults.SubtreeRead = null; + root.Faults.CheckpointWritten = null; + } + + Exception? membershipFailure = await Record.ExceptionAsync(() => AssertFileMoveMembershipAsync(root, scenario)); + // Defer the membership assertion so the baseline also exercises the user-visible + // parent rollback/rescan rather than stopping at its stale checkpoint. + await AssertFileMoveParentRecoveryAsync(root, scenario); + Assert.Null(membershipFailure); + foreach ((Guid id, CloudStateCheckpoint[] image) in scenario.Ancestors) + { + CloudDirectoryProvenance provenance = CloudDirectoryProvenance.Decode(image[0].Value); + Assert.Equal(1, reads.GetValueOrDefault(provenance.RelativePath)); + Assert.Equal(1, writes.GetValueOrDefault(CloudDirectoryProvenance.MembersName(id))); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task FileMoveMembershipFaultRollsBackAndRestartRetryPreservesPendingJournal(bool viaFeed) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + FileMoveMembershipScenario scenario = await PrepareFileMoveMembershipAsync(root, "cross"); + CloudLocalFileBinding? originalBinding = (await root.FileSystem.GetFile(scenario.Source).InspectAsync()).LocalBinding; + Assert.NotNull(originalBinding); + IOException failure = new("File move ancestor membership write fault."); + string failedMemberName = scenario.Ancestors.Values.First(image => + CloudDirectoryProvenance.Decode(image[0].Value).RelativePath == "Docs\\Left")[1].Name; + bool reached = false; + root.Faults.CheckpointWritten = name => + { + if (name == failedMemberName) + { + reached = true; + root.Faults.CheckpointWritten = null; + throw failure; + } + }; + try + { + if (viaFeed) + { + File.Move(Path.Combine(root.RootPath, scenario.Source), Path.Combine(root.RootPath, scenario.Target)); + await ProcessRenameAsync(root, scenario.Source, scenario.Target); + } + else + { + CloudItemCoordinationException error = await Assert.ThrowsAsync(() => + root.FileSystem.GetFile(scenario.Source).MoveToAsync( + root.FileSystem.GetDirectory("Docs/Right"), "renamed.txt").AsTask()); + Assert.Same(failure, error.InnerException); + } + } + finally + { + root.Faults.CheckpointWritten = null; + } + + Assert.True(reached); + Assert.False(File.Exists(Path.Combine(root.RootPath, scenario.Source))); + Assert.True(File.Exists(Path.Combine(root.RootPath, scenario.Target))); + CloudOperationJournalEntry? unresolved = null; + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + Assert.Equal(scenario.Source, (await verify.Items.GetByItemIdAsync(scenario.FileId))!.RelativePath); + foreach (CloudStateCheckpoint[] image in scenario.Ancestors.Values) + { + await AssertCheckpointImageAsync(verify, image); + } + + await AssertCheckpointImageAsync(verify, scenario.History); + await AssertFileMoveJournalImageAsync(verify, scenario.Pending); + if (viaFeed) + { + unresolved = Assert.Single((await verify.Operations.ListAsync(100)).Where(row => row.Kind == CloudStateOperationKind.Move)); + Assert.Null(unresolved.ItemId); + Assert.True(LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + } + else + { + Assert.Single(await verify.Operations.ListAsync(100)); + } + } + + await root.RestartAsync(); + if (viaFeed) + { + await ProcessRenameAsync(root, scenario.Source, scenario.Target); + } + else + { + await root.FileSystem.GetFile(scenario.Source).MoveToAsync(root.FileSystem.GetDirectory("Docs/Right"), "renamed.txt"); + } + + Assert.Equal(originalBinding, (await root.FileSystem.GetFile(scenario.Target).InspectAsync()).LocalBinding); + await AssertFileMoveMembershipAsync(root, scenario); + if (viaFeed) + { + // A real storage failure retains its fence until application reconciliation. A + // successful replay refreshes membership but never ACKs the original observation. + await using CloudLocalChangeFeed reconciled = CloudLocalChangeFeed.CreateForTesting(root.RootPath, root.Store, new(), new FeedSource()); + await reconciled.StartAsync(); + Assert.True((await reconciled.BeginScanAsync()).RequiresFullRescan); + await reconciled.AcknowledgeFullRescanAsync(); + } + + await AssertFileMoveParentRecoveryAsync(root, scenario); + await using ICloudStateTransaction after = await root.Store.BeginTransactionAsync(); + if (unresolved is not null) + { + await AssertFileMoveJournalImageAsync(after, unresolved); + } + } + + private static async Task PrepareFileMoveMembershipAsync(DirectoryMoveTestRoot root, string shape) + { + CloudDirectoryPlaceholderSpec left = CloudDirectoryPlaceholderSpec.CreateBuilder("Left", "file-left") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build(); + CloudDirectoryPlaceholderSpec right = CloudDirectoryPlaceholderSpec.CreateBuilder("Right", "file-right") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build(); + CloudDirectory docs = root.FileSystem.GetDirectory("Docs"); + await docs.CreatePlaceholdersAsync(shape == "cross" ? [left, right] : [left]); + string source = shape == "in" ? "outside.txt" : "Docs\\Left\\child.txt"; + string target = shape switch + { + "cross" => "Docs\\Right\\renamed.txt", + "out" => "outside.txt", + "case" => "Docs\\Left\\CHILD.TXT", + _ => "Docs\\Left\\renamed.txt", + }; + CloudFilePlaceholderSpec file = CloudFilePlaceholderSpec.CreateBuilder(Path.GetFileName(source), "file-movable", 0).Build(); + await root.FileSystem.GetDirectory(Path.GetDirectoryName(source)!).CreatePlaceholderAsync(file); + Guid deletedId = Guid.NewGuid(); + CloudOperationJournalEntry pending; + await using (ICloudStateTransaction seed = await root.Store.BeginTransactionAsync()) + { + CloudItemState current = (await seed.Items.GetByItemIdAsync(file.Identity.ItemId))!; + await seed.Items.UpsertAsync(new(current.ItemId, current.RemoteId, current.RelativePath, current.Kind, + "file-current", 33, false, current.UpdatedAt)); + await seed.Items.UpsertAsync(new(deletedId, "file-deleted", "Docs\\Left\\deleted.txt", CloudItemKind.File, + "file-deleted-revision", 42, true, DateTimeOffset.UtcNow)); + pending = await seed.Operations.EnqueueAsync(new(Guid.NewGuid(), CloudStateOperationKind.ContentUpdate, current.ItemId, + new LocalChangePayload(source, null, false, DateTimeOffset.UtcNow).Encode(), DateTimeOffset.UtcNow)); + await seed.CommitAsync(); + } + + CloudDirectoryMoveProof historical = await docs.PrepareMoveAsync(root.FileSystem.Root, "Archived"); + await docs.MoveToAsync(root.FileSystem.Root, "Archived", new CloudMoveOptions(historical)); + await root.FileSystem.GetDirectory("Archived").MoveToAsync(root.FileSystem.Root, "Docs"); + Dictionary ancestors = new(); + await using ICloudStateTransaction read = await root.Store.BeginTransactionAsync(); + foreach (Guid id in new[] { root.DirectoryIdentity.ItemId, left.Identity.ItemId }.Concat(shape == "cross" ? [right.Identity.ItemId] : Array.Empty())) + { + ancestors.Add(id, [(await read.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(id)))!, + (await read.Checkpoints.GetAsync(CloudDirectoryProvenance.MembersName(id)))!]); + } + + CloudStateCheckpoint[] history = (await read.Checkpoints.ListAsync($"cfsharp/namespace/preparations/{historical.ProofId:N}")).ToArray(); + Assert.Equal(3, history.Length); + return new(source, target, file.Identity.ItemId, deletedId, pending, ancestors, history); + } + + private static async Task AssertFileMoveMembershipAsync(DirectoryMoveTestRoot root, FileMoveMembershipScenario scenario) + { + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + Assert.Equal(scenario.Target, (await verify.Items.GetByItemIdAsync(scenario.FileId))!.RelativePath); + foreach ((Guid id, CloudStateCheckpoint[] image) in scenario.Ancestors) + { + await AssertCheckpointImageAsync(verify, [image[0]]); + CloudDirectoryProvenance provenance = CloudDirectoryProvenance.Decode(image[0].Value); + IReadOnlyList members = CloudDirectoryMoveEvidence.DecodeMembers( + (await verify.Checkpoints.GetAsync(image[1].Name))!.Value, provenance.EvidenceId, id); + IReadOnlyList rows = await verify.Items.ListSubtreeAsync(provenance.RelativePath); + Assert.Equal(rows.Select(item => (item.ItemId, item.Kind, + CloudDirectoryStateProjection.MapPath(item.RelativePath, provenance.RelativePath, string.Empty))).OrderBy(item => item.ItemId), + members.Select(member => (member.ItemId, member.Kind, member.Suffix)).OrderBy(member => member.ItemId)); + } + + await AssertCheckpointImageAsync(verify, scenario.History); + await AssertFileMoveJournalImageAsync(verify, scenario.Pending); + } + + private static async Task AssertFileMoveParentRecoveryAsync(DirectoryMoveTestRoot root, FileMoveMembershipScenario scenario) + { + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + await ProcessRenameAsync(root, "Docs", "Moved"); + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + Assert.Equal("Moved", (await verify.Items.GetByItemIdAsync(root.DirectoryIdentity.ItemId))!.RelativePath); + CloudItemState file = (await verify.Items.GetByItemIdAsync(scenario.FileId))!; + Assert.Equal(scenario.Target.StartsWith("Docs\\", StringComparison.Ordinal) + ? CloudDirectoryStateProjection.MapPath(scenario.Target, "Docs", "Moved") : scenario.Target, file.RelativePath); + Assert.Equal("file-current", file.RemoteRevision); + Assert.Equal(33, file.LocalFileId); + CloudItemState deleted = (await verify.Items.GetByItemIdAsync(scenario.DeletedId))!; + Assert.Equal("Moved\\Left\\deleted.txt", deleted.RelativePath); + Assert.True(deleted.IsTombstone); + Assert.Equal("file-deleted-revision", deleted.RemoteRevision); + Assert.Equal(42, deleted.LocalFileId); + await AssertCheckpointImageAsync(verify, scenario.History); + await AssertFileMoveJournalImageAsync(verify, scenario.Pending); + } + + await using CloudLocalChangeFeed delivery = CloudLocalChangeFeed.CreateForTesting(root.RootPath, root.Store, new(), new FeedSource()); + await delivery.StartAsync(); + CloudLocalChangeBatch batch = await delivery.ReadBatchAsync(); + Assert.False(batch.RequiresFullRescan); + CloudLocalChangeScan scan = await delivery.BeginScanAsync(); + Assert.False(scan.RequiresFullRescan); + CloudLocalChangePage page = await delivery.ReadPageAsync(scan, 0, 100); + Assert.False(page.RequiresFullRescan); + Assert.Equal(batch.Changes.Select(change => change.OperationId), page.Changes.Select(change => change.OperationId)); + Assert.Contains(page.Changes, change => change.ItemId == root.DirectoryIdentity.ItemId && change.Kind == CloudLocalChangeKind.Move); + } + + private static async Task AssertFileMoveJournalImageAsync(ICloudStateTransaction transaction, CloudOperationJournalEntry image) + { + CloudOperationJournalEntry actual = (await transaction.Operations.GetAsync(image.OperationId))!; + Assert.Equal(image.Sequence, actual.Sequence); + Assert.Equal(image.Kind, actual.Kind); + Assert.Equal(image.ItemId, actual.ItemId); + Assert.Equal(image.CreatedAt, actual.CreatedAt); + Assert.Equal(image.Payload.ToArray(), actual.Payload.ToArray()); + } + + private sealed record FileMoveMembershipScenario(string Source, string Target, Guid FileId, Guid DeletedId, + CloudOperationJournalEntry Pending, IReadOnlyDictionary Ancestors, CloudStateCheckpoint[] History); +} From cb4c6bc23955d6798c1ba5ff3203a42c2c7d3df2 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:15:04 +0800 Subject: [PATCH 37/40] fix(local-changes): refresh membership for discovered items --- docs/placeholders.md | 6 + .../LocalChanges/CloudLocalChangeFeed.cs | 10 +- .../Persistence/CloudDirectoryProvenance.cs | 8 + .../CloudFeedDiscoveryProvenanceTests.cs | 180 ++++++++++++++++++ 4 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudFeedDiscoveryProvenanceTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index 3cfcd87..8e94afb 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -27,6 +27,12 @@ prepared proofs remain unchanged. Removing an identity or reverting a directory provenance. Storage without complete IDs retains existing creation behavior but cannot establish this recovery evidence; legacy directories can explicitly prepare while still at their source. +When the local-change feed first discovers a file or directory through creation or another +observation, its new item row, journal entry and captured ancestors' live membership commit +together. It does not infer a managed binding for an ordinary local item or modify any immutable +prepared manifest. A later parent move can project these known rows; ordinary native children +still retain their separate full-reconciliation requirement. + Known file moves and renames through `MoveToAsync` or the local-change feed update official paths and refresh retained live membership of both source and destination ancestors in one transaction. Shared ancestors are refreshed once, including for case-only renames and files moved into or out diff --git a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs index 605c4b2..d8127c6 100644 --- a/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs +++ b/src/CfSharp/LocalChanges/CloudLocalChangeFeed.cs @@ -22,6 +22,8 @@ namespace CfSharp; /// Known file rename paths, retained source/destination ancestor membership, and the new journal /// observation commit in one transaction. Ancestor native bindings, immutable directory recovery /// evidence and existing pending operations remain unchanged. Shared ancestors are refreshed once. +/// First discovery of a local file or directory also refreshes captured ancestor membership in +/// its item/journal transaction, without inferring a managed native binding for that local item. /// /// public sealed partial class CloudLocalChangeFeed : IDisposable, IAsyncDisposable @@ -844,8 +846,9 @@ await UpsertCheckpointAsync( ? current : previous ?? current; Guid? itemId = uncertainDirectory is not null ? null : directoryProjection?.ItemId ?? state?.ItemId; + bool discovered = state is null && kind != CloudLocalChangeKind.Delete; - if (uncertainDirectory is null && directoryProjection is null && state is null && kind != CloudLocalChangeKind.Delete) + if (uncertainDirectory is null && directoryProjection is null && discovered) { itemId = Guid.NewGuid(); state = new CloudItemState( @@ -876,6 +879,11 @@ await UpsertCheckpointAsync( await CloudDirectoryProvenance.RefreshMoveAncestorsAsync(transaction, previousPath.Value.RelativePath, path.RelativePath, cancellationToken).ConfigureAwait(false); } + else if (discovered) + { + await CloudDirectoryProvenance.RefreshProjectionAncestorsAsync(transaction, path.RelativePath, + cancellationToken).ConfigureAwait(false); + } } LocalChangePayload payload = new( diff --git a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs index bcc36bf..ab683d4 100644 --- a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs +++ b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs @@ -214,6 +214,14 @@ await RefreshMoveAncestorsAsync(transaction, proof.SourceRelativePath, proof.Des cancellationToken).ConfigureAwait(false); } + internal static async ValueTask RefreshProjectionAncestorsAsync(ICloudStateTransaction transaction, + string relativePath, CancellationToken cancellationToken) + { + HashSet ancestors = new(StringComparer.OrdinalIgnoreCase); + AddAncestors(ancestors, relativePath); + await RefreshAncestorsAsync(transaction, ancestors, cancellationToken).ConfigureAwait(false); + } + internal static async ValueTask RefreshMoveAncestorsAsync(ICloudStateTransaction transaction, string sourceRelativePath, string destinationRelativePath, CancellationToken cancellationToken) { diff --git a/tests/CfSharp.IntegrationTests/CloudFeedDiscoveryProvenanceTests.cs b/tests/CfSharp.IntegrationTests/CloudFeedDiscoveryProvenanceTests.cs new file mode 100644 index 0000000..b58eb78 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudFeedDiscoveryProvenanceTests.cs @@ -0,0 +1,180 @@ +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData(false, false, false)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] + [InlineData(false, true, true)] + [InlineData(true, false, false)] + [InlineData(true, false, true)] + [InlineData(true, true, false)] + [InlineData(true, true, true)] + public async Task FeedDiscoveryKeepsKnownLocalMembersRecoverableAfterParentRename( + bool directory, bool modified, bool explicitPreparation) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + DiscoveryProvenanceScenario scenario = await PrepareDiscoveryProvenanceAsync(root, directory); + await ProcessDiscoveryAsync(root, scenario.Path, modified); + CloudItemState discovered; + CloudOperationJournalEntry observation; + Exception? membershipFailure = null; + await using (ICloudStateTransaction read = await root.Store.BeginTransactionAsync()) + { + discovered = (await read.Items.GetByRelativePathAsync(scenario.Path))!; + Assert.NotNull(discovered); + Assert.Equal(directory ? CloudItemKind.Directory : CloudItemKind.File, discovered.Kind); + observation = Assert.Single((await read.Operations.ListAsync(100)).Where(row => row.ItemId == discovered.ItemId)); + await AssertCheckpointImageAsync(read, [scenario.Live[0], .. scenario.History]); + await AssertFileMoveJournalImageAsync(read, scenario.Pending); + Assert.Null(await read.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(discovered.ItemId))); + if (!explicitPreparation) + { + membershipFailure = await Record.ExceptionAsync(() => AssertDiscoveryMembersAsync(read, scenario, discovered)); + } + } + + if (explicitPreparation) + { + await root.FileSystem.GetDirectory("Docs").PrepareMoveAsync(root.FileSystem.Root, "Moved"); + } + + // Exercise the visible recovery before asserting the retained manifest, so the + // baseline exposes the rolled-back subtree while the explicit control succeeds. + await AssertDiscoveredParentRecoveryAsync(root, scenario, discovered, observation); + Assert.Null(membershipFailure); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task FeedDiscoveryMembershipFaultRollsBackAndRestartRetryPreservesHistory(bool directory) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + DiscoveryProvenanceScenario scenario = await PrepareDiscoveryProvenanceAsync(root, directory); + bool reached = false; + root.Faults.CheckpointWritten = name => + { + if (name == scenario.Live[1].Name) + { + reached = true; + root.Faults.CheckpointWritten = null; + throw new IOException("Discovered item ancestor membership write fault."); + } + }; + try + { + await ProcessDiscoveryAsync(root, scenario.Path, modified: false); + } + finally + { + root.Faults.CheckpointWritten = null; + } + + Assert.True(reached); + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + Assert.Null(await verify.Items.GetByRelativePathAsync(scenario.Path)); + await AssertCheckpointImageAsync(verify, [.. scenario.Live, .. scenario.History]); + await AssertFileMoveJournalImageAsync(verify, scenario.Pending); + Assert.Single(await verify.Operations.ListAsync(100)); + Assert.True(LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + } + + await root.RestartAsync(); + await ProcessDiscoveryAsync(root, scenario.Path, modified: false); + CloudItemState discovered; + CloudOperationJournalEntry observation; + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + discovered = (await verify.Items.GetByRelativePathAsync(scenario.Path))!; + await AssertDiscoveryMembersAsync(verify, scenario, discovered); + await AssertCheckpointImageAsync(verify, [scenario.Live[0], .. scenario.History]); + observation = Assert.Single((await verify.Operations.ListAsync(100)).Where(row => row.ItemId == discovered.ItemId)); + } + + // Successful retry cannot clear the durable fence from the actual storage failure. + await using (CloudLocalChangeFeed reconciled = CloudLocalChangeFeed.CreateForTesting(root.RootPath, root.Store, new(), new FeedSource())) + { + await reconciled.StartAsync(); + Assert.True((await reconciled.BeginScanAsync()).RequiresFullRescan); + await reconciled.AcknowledgeFullRescanAsync(); + } + + await AssertDiscoveredParentRecoveryAsync(root, scenario, discovered, observation); + } + + private static async Task PrepareDiscoveryProvenanceAsync(DirectoryMoveTestRoot root, bool directory) + { + CloudOperationJournalEntry pending = await SeedPendingChildAsync(root); + CloudDirectory docs = root.FileSystem.GetDirectory("Docs"); + CloudDirectoryMoveProof historical = await docs.PrepareMoveAsync(root.FileSystem.Root, "Archived"); + await docs.MoveToAsync(root.FileSystem.Root, "Archived", new CloudMoveOptions(historical)); + await root.FileSystem.GetDirectory("Archived").MoveToAsync(root.FileSystem.Root, "Docs"); + string path = directory ? "Docs\\Local" : "Docs\\local.txt"; + if (directory) + { + Directory.CreateDirectory(Path.Combine(root.RootPath, path)); + } + else + { + await File.WriteAllBytesAsync(Path.Combine(root.RootPath, path), []); + } + + await using ICloudStateTransaction read = await root.Store.BeginTransactionAsync(); + CloudStateCheckpoint[] live = [(await read.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(root.DirectoryIdentity.ItemId)))!, + (await read.Checkpoints.GetAsync(CloudDirectoryProvenance.MembersName(root.DirectoryIdentity.ItemId)))!]; + CloudStateCheckpoint[] history = (await read.Checkpoints.ListAsync($"cfsharp/namespace/preparations/{historical.ProofId:N}")).ToArray(); + Assert.Equal(3, history.Length); + return new(path, pending, live, history); + } + + private static async Task ProcessDiscoveryAsync(DirectoryMoveTestRoot root, string path, bool modified) + { + FeedSource source = new(); + await using CloudLocalChangeFeed feed = CloudLocalChangeFeed.CreateForTesting(root.RootPath, root.Store, new(), source); + await feed.StartAsync(); + await source.EmitAsync(new(modified ? LocalChangeSourceAction.Modified : LocalChangeSourceAction.Created, path)); + await feed.DisposeAsync(); + } + + private static async Task AssertDiscoveryMembersAsync(ICloudStateTransaction transaction, + DiscoveryProvenanceScenario scenario, CloudItemState discovered) + { + CloudDirectoryProvenance provenance = CloudDirectoryProvenance.Decode(scenario.Live[0].Value); + IReadOnlyList members = CloudDirectoryMoveEvidence.DecodeMembers( + (await transaction.Checkpoints.GetAsync(scenario.Live[1].Name))!.Value, provenance.EvidenceId, provenance.RootItemId); + Assert.Contains(members, member => member.ItemId == discovered.ItemId && member.Kind == discovered.Kind && + member.Suffix == Path.GetFileName(scenario.Path)); + } + + private static async Task AssertDiscoveredParentRecoveryAsync(DirectoryMoveTestRoot root, DiscoveryProvenanceScenario scenario, + CloudItemState discovered, CloudOperationJournalEntry observation) + { + Directory.Move(Path.Combine(root.RootPath, "Docs"), Path.Combine(root.RootPath, "Moved")); + await ProcessRenameAsync(root, "Docs", "Moved"); + await AssertProjectedChildAsync(root, "Moved", scenario.Pending); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + CloudItemState current = (await verify.Items.GetByItemIdAsync(discovered.ItemId))!; + Assert.Equal("Moved\\" + Path.GetFileName(scenario.Path), current.RelativePath); + Assert.Equal(discovered.RemoteId, current.RemoteId); + Assert.Equal(discovered.Kind, current.Kind); + Assert.Null(current.LocalFileId); + Assert.Null(current.RemoteRevision); + Assert.False(current.IsTombstone); + await AssertCheckpointImageAsync(verify, scenario.History); + await AssertFileMoveJournalImageAsync(verify, observation); + CloudOperationJournalEntry move = Assert.Single((await verify.Operations.ListAsync(100)).Where(row => row.Kind == CloudStateOperationKind.Move)); + Assert.Equal(root.DirectoryIdentity.ItemId, move.ItemId); + // Ordinary native children still require reconciliation even after all known rows + // project successfully. Updating membership must not weaken that separate fence. + Assert.True(LocalChangeCheckpoint.Decode((await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName))!.Value).RequiresFullRescan); + } + + private sealed record DiscoveryProvenanceScenario(string Path, CloudOperationJournalEntry Pending, + CloudStateCheckpoint[] Live, CloudStateCheckpoint[] History); +} From 019a39dc806eb96053917fc2ed47ddd558a3185c Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:17:55 +0800 Subject: [PATCH 38/40] fix(persistence): relocate live provenance under ordinary moves --- docs/placeholders.md | 5 + .../CloudFileSystem.NamespaceOperations.cs | 4 +- .../Items/CloudItem.NamespaceOperations.cs | 3 + .../Persistence/CloudDirectoryProvenance.cs | 34 +++ .../CloudOrdinaryMoveProvenanceTests.cs | 207 ++++++++++++++++++ 5 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudOrdinaryMoveProvenanceTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index 8e94afb..f816c84 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -70,6 +70,11 @@ source/destination after a crash uses the retained indexed preparation and perfo projection. To select a specific externally prepared proof, pass `new CloudMoveOptions(proof)`; it must name that exact target. Missing-source directory retries without evidence fail closed. Normal ordinary-directory moves and the existing boolean file-replacement options remain available. +Moving an ordinary ancestor relocates any existing managed descendants' live binding paths and +rebuilds their mutable membership in the same official subtree transaction. It retains their +native bindings and immutable recovery history, so later external descendant moves can use +pre-rename provenance, including after restart. Directories without captured bindings acquire no +recovery evidence from this path; absent ordinary sources still require historical pre-move proof. Directory move results and coordination exceptions expose `DirectoryReconciliation`, preserving a post-commit race or rescan requirement without inventing a failed commit. The bounded intent index may select a newer preparation but never replaces previous immutable proofs or receipts. diff --git a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs index 495c29c..4bf899f 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs @@ -433,8 +433,8 @@ private static async ValueTask PersistMovedStateAsync( .ConfigureAwait(false); await CloudDirectoryStateProjection.ProjectAsync(transaction, sourceEntries, sourceRelativePath, destinationRelativePath, DateTimeOffset.UtcNow, cancellationToken).ConfigureAwait(false); - await CloudDirectoryProvenance.RefreshMoveAncestorsAsync(transaction, sourceRelativePath, - destinationRelativePath, cancellationToken).ConfigureAwait(false); + await CloudDirectoryProvenance.RelocateRetainedPathsAsync(transaction, sourceEntries, + sourceRelativePath, destinationRelativePath, cancellationToken).ConfigureAwait(false); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); } diff --git a/src/CfSharp/Items/CloudItem.NamespaceOperations.cs b/src/CfSharp/Items/CloudItem.NamespaceOperations.cs index b69e3b8..a955155 100644 --- a/src/CfSharp/Items/CloudItem.NamespaceOperations.cs +++ b/src/CfSharp/Items/CloudItem.NamespaceOperations.cs @@ -27,6 +27,9 @@ public abstract partial class CloudItem /// preparations and receipts, and existing pending journal operations remain unchanged. /// A failed file projection can be retried on the original source reference and destination, /// including after restart, without issuing another move when the source is already absent. + /// An ordinary directory move also relocates existing managed descendants' live provenance + /// and rebuilds their membership in that path transaction, retaining their native identities + /// and immutable recovery history. It does not create evidence for an ordinary directory. /// /// /// The destination belongs to another file system, the name is invalid, or replacement was diff --git a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs index ab683d4..ac1db99 100644 --- a/src/CfSharp/Persistence/CloudDirectoryProvenance.cs +++ b/src/CfSharp/Persistence/CloudDirectoryProvenance.cs @@ -214,6 +214,40 @@ await RefreshMoveAncestorsAsync(transaction, proof.SourceRelativePath, proof.Des cancellationToken).ConfigureAwait(false); } + internal static async ValueTask RelocateRetainedPathsAsync(ICloudStateTransaction transaction, + IReadOnlyList sourceEntries, string sourceRelativePath, + string destinationRelativePath, CancellationToken cancellationToken) + { + Guid? scope = null; + foreach (CloudItemState item in sourceEntries.Where(item => item.Kind == CloudItemKind.Directory && !item.IsTombstone)) + { + CloudStateCheckpoint? checkpoint = await transaction.Checkpoints.GetAsync(BindingName(item.ItemId), cancellationToken).ConfigureAwait(false); + if (checkpoint is null) + { + continue; + } + + scope ??= await CloudDirectoryMoveEvidence.GetScopeAsync(transaction, false, cancellationToken).ConfigureAwait(false); + CloudDirectoryProvenance provenance = Decode(checkpoint.Value); + string targetPath = CloudDirectoryStateProjection.MapPath(item.RelativePath, sourceRelativePath, destinationRelativePath); + if (provenance.RootItemId != item.ItemId || provenance.StoreScope != scope || + (!CloudDirectoryStateProjection.SamePath(provenance.RelativePath, item.RelativePath) && + !CloudDirectoryStateProjection.SamePath(provenance.RelativePath, targetPath))) + { + throw new CloudDirectoryProjectionConflictException("Retained directory provenance belongs to another namespace location."); + } + + // Moving an ordinary ancestor changes existing managed descendants' locations, + // not their native identities. Rebuild only mutable records from final item rows; + // directories without retained bindings must not acquire fabricated provenance. + await RetainMetadataAsync(transaction, provenance with { EvidenceId = Guid.NewGuid(), RelativePath = targetPath }, + cancellationToken).ConfigureAwait(false); + } + + await RefreshMoveAncestorsAsync(transaction, sourceRelativePath, destinationRelativePath, + cancellationToken).ConfigureAwait(false); + } + internal static async ValueTask RefreshProjectionAncestorsAsync(ICloudStateTransaction transaction, string relativePath, CancellationToken cancellationToken) { diff --git a/tests/CfSharp.IntegrationTests/CloudOrdinaryMoveProvenanceTests.cs b/tests/CfSharp.IntegrationTests/CloudOrdinaryMoveProvenanceTests.cs new file mode 100644 index 0000000..a2cfca5 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudOrdinaryMoveProvenanceTests.cs @@ -0,0 +1,207 @@ +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData("rename", false)] + [InlineData("rename", true)] + [InlineData("cross", false)] + [InlineData("cross", true)] + [InlineData("case", false)] + [InlineData("case", true)] + public async Task OrdinaryParentMoveRetainsManagedDescendantRecovery(string shape, bool facadeRecovery) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + OrdinaryMoveProvenanceScenario scenario = await PrepareOrdinaryMoveProvenanceAsync(root, nested: false); + string target = shape switch { "cross" => "Container\\Relocated", "case" => "ORDINARY", _ => "RenamedOrdinary" }; + Directory.CreateDirectory(Path.Combine(root.RootPath, "Container")); + CloudItemMoveResult result = await root.FileSystem.GetDirectory("Ordinary").MoveToAsync( + root.FileSystem.GetDirectory(Path.GetDirectoryName(target)!), Path.GetFileName(target)); + Assert.Null(result.DirectoryReconciliation); + Assert.Equal(3, result.DurableStateEntriesUpdated); + Exception? provenanceFailure = null; + if (!facadeRecovery) + { + provenanceFailure = await Record.ExceptionAsync(() => AssertOrdinaryMoveProvenanceAsync(root, scenario, target)); + } + + await root.RestartAsync(); + await AssertNestedManagedRecoveryAsync(root, scenario, target, facadeRecovery); + Assert.Null(provenanceFailure); + } + + [Fact] + public async Task OrdinaryParentMoveRelocatesAllNestedLiveBindingsBeforeFeedRecovery() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + OrdinaryMoveProvenanceScenario scenario = await PrepareOrdinaryMoveProvenanceAsync(root, nested: true); + await root.FileSystem.GetDirectory("Ordinary").MoveToAsync(root.FileSystem.Root, "RenamedOrdinary"); + Exception? provenanceFailure = await Record.ExceptionAsync(() => AssertOrdinaryMoveProvenanceAsync(root, scenario, "RenamedOrdinary")); + await root.RestartAsync(); + await AssertNestedManagedRecoveryAsync(root, scenario, "RenamedOrdinary", facadeRecovery: false); + Assert.Null(provenanceFailure); + } + + [Fact] + public async Task OrdinaryParentMembershipFaultRollsBackPathsAndBindingsBeforeRestartRetry() + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + OrdinaryMoveProvenanceScenario scenario = await PrepareOrdinaryMoveProvenanceAsync(root, nested: true); + IOException failure = new("Ordinary parent nested membership write fault."); + string failedName = scenario.Live.Last()[1].Name; + bool reached = false; + root.Faults.CheckpointWritten = name => + { + if (name == failedName) + { + reached = true; + root.Faults.CheckpointWritten = null; + throw failure; + } + }; + try + { + CloudItemCoordinationException error = await Assert.ThrowsAsync(() => + root.FileSystem.GetDirectory("Ordinary").MoveToAsync(root.FileSystem.Root, "RenamedOrdinary").AsTask()); + Assert.Same(failure, error.InnerException); + } + finally + { + root.Faults.CheckpointWritten = null; + } + + Assert.True(reached); + Assert.False(Directory.Exists(Path.Combine(root.RootPath, "Ordinary"))); + Assert.True(Directory.Exists(Path.Combine(root.RootPath, "RenamedOrdinary"))); + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + foreach (CloudItemState row in scenario.Rows) + { + CloudItemState current = (await verify.Items.GetByItemIdAsync(row.ItemId))!; + Assert.Equal((row.RemoteId, row.RelativePath, row.Kind, row.RemoteRevision, row.LocalFileId, row.IsTombstone, row.UpdatedAt), + (current.RemoteId, current.RelativePath, current.Kind, current.RemoteRevision, current.LocalFileId, current.IsTombstone, current.UpdatedAt)); + } + + await AssertCheckpointImageAsync(verify, scenario.Live.SelectMany(image => image).Concat(scenario.History)); + await AssertFileMoveJournalImageAsync(verify, scenario.Pending); + Assert.Single(await verify.Operations.ListAsync(100)); + } + + await root.RestartAsync(); + // Ordinary directories still cannot recover an absent source without pre-move proof. + // Restore its native source explicitly, then retry the original authorized operation. + await Assert.ThrowsAsync(() => root.FileSystem.GetDirectory("Ordinary") + .MoveToAsync(root.FileSystem.Root, "RenamedOrdinary").AsTask()); + Directory.Move(Path.Combine(root.RootPath, "RenamedOrdinary"), Path.Combine(root.RootPath, "Ordinary")); + await root.FileSystem.GetDirectory("Ordinary").MoveToAsync(root.FileSystem.Root, "RenamedOrdinary"); + await AssertOrdinaryMoveProvenanceAsync(root, scenario, "RenamedOrdinary"); + await AssertNestedManagedRecoveryAsync(root, scenario, "RenamedOrdinary", facadeRecovery: false); + } + + private static async Task PrepareOrdinaryMoveProvenanceAsync(DirectoryMoveTestRoot root, bool nested) + { + CloudDirectory docs = root.FileSystem.GetDirectory("Docs"); + if (nested) + { + await docs.CreatePlaceholderAsync(CloudDirectoryPlaceholderSpec.CreateBuilder("Inner", "ordinary-inner") + .WithPopulationState(CloudDirectoryPopulationState.Complete).Build()); + await root.FileSystem.GetDirectory("Docs/Inner").CreatePlaceholderAsync( + CloudFilePlaceholderSpec.CreateBuilder("leaf.txt", "ordinary-leaf", 0).Build()); + } + + CloudOperationJournalEntry pending = await SeedPendingChildAsync(root); + await using (ICloudStateTransaction seed = await root.Store.BeginTransactionAsync()) + { + await seed.Items.UpsertAsync(new(Guid.NewGuid(), "ordinary-deleted", "Docs\\deleted.txt", CloudItemKind.File, + "deleted-revision", 456, true, DateTimeOffset.UtcNow)); + await seed.CommitAsync(); + } + + await docs.MoveToAsync(root.FileSystem.Root, "Archived"); + await root.FileSystem.GetDirectory("Archived").MoveToAsync(root.FileSystem.Root, "Docs"); + await docs.MoveToAsync(root.FileSystem.GetDirectory("Ordinary"), "Docs"); + await using ICloudStateTransaction read = await root.Store.BeginTransactionAsync(); + CloudItemState[] rows = (await read.Items.ListSubtreeAsync("Ordinary")).ToArray(); + List live = []; + foreach (CloudItemState row in rows.Where(item => item.Kind == CloudItemKind.Directory && !item.IsTombstone)) + { + live.Add([(await read.Checkpoints.GetAsync(CloudDirectoryProvenance.BindingName(row.ItemId)))!, + (await read.Checkpoints.GetAsync(CloudDirectoryProvenance.MembersName(row.ItemId)))!]); + } + + CloudStateCheckpoint[] history = (await read.Checkpoints.ListAsync("cfsharp/namespace/preparations/")).ToArray(); + Assert.Equal(nested ? 2 : 1, live.Count); + Assert.Equal(9, history.Length); + return new(rows, live.ToArray(), history, pending); + } + + private static async Task AssertOrdinaryMoveProvenanceAsync(DirectoryMoveTestRoot root, + OrdinaryMoveProvenanceScenario scenario, string target) + { + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + foreach (CloudStateCheckpoint[] image in scenario.Live) + { + CloudDirectoryProvenance before = CloudDirectoryProvenance.Decode(image[0].Value); + CloudDirectoryProvenance current = CloudDirectoryProvenance.Decode((await verify.Checkpoints.GetAsync(image[0].Name))!.Value); + Assert.Equal(CloudDirectoryStateProjection.MapPath(before.RelativePath, "Ordinary", target), current.RelativePath); + Assert.Equal(before.RootItemId, current.RootItemId); + Assert.Equal(before.StoreScope, current.StoreScope); + Assert.Equal(before.Binding, current.Binding); + Assert.Equal(before.Identity, current.Identity); + IReadOnlyList members = CloudDirectoryMoveEvidence.DecodeMembers( + (await verify.Checkpoints.GetAsync(image[1].Name))!.Value, current.EvidenceId, current.RootItemId); + IReadOnlyList rows = await verify.Items.ListSubtreeAsync(current.RelativePath); + Assert.Equal(rows.Select(item => (item.ItemId, item.Kind, + CloudDirectoryStateProjection.MapPath(item.RelativePath, current.RelativePath, string.Empty))).OrderBy(item => item.ItemId), + members.Select(member => (member.ItemId, member.Kind, member.Suffix)).OrderBy(member => member.ItemId)); + } + + await AssertCheckpointImageAsync(verify, scenario.History); + await AssertFileMoveJournalImageAsync(verify, scenario.Pending); + } + + private static async Task AssertNestedManagedRecoveryAsync(DirectoryMoveTestRoot root, + OrdinaryMoveProvenanceScenario scenario, string parent, bool facadeRecovery) + { + string source = Path.Combine(parent, "Docs"); + Assert.Equal(CloudDirectoryProvenance.Decode(scenario.Live[0][0].Value).Binding, + (await root.FileSystem.GetDirectory(source).InspectAsync()).LocalBinding); + if (facadeRecovery) + { + CloudItemMoveResult result = await root.FileSystem.GetDirectory(source).MoveToAsync(root.FileSystem.Root, "Recovered"); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, result.DirectoryReconciliation!.Outcome); + } + else + { + Directory.Move(Path.Combine(root.RootPath, source), Path.Combine(root.RootPath, "Recovered")); + await ProcessRenameAsync(root, source, "Recovered"); + } + + await AssertProjectedChildAsync(root, "Recovered", scenario.Pending); + await using ICloudStateTransaction verify = await root.Store.BeginTransactionAsync(); + foreach (CloudItemState row in scenario.Rows) + { + CloudItemState current = (await verify.Items.GetByItemIdAsync(row.ItemId))!; + Assert.Equal(CloudDirectoryStateProjection.MapPath(row.RelativePath, "Ordinary\\Docs", "Recovered"), current.RelativePath); + Assert.Equal(row.RemoteId, current.RemoteId); + Assert.Equal(row.RemoteRevision, current.RemoteRevision); + Assert.Equal(row.LocalFileId, current.LocalFileId); + Assert.Equal(row.IsTombstone, current.IsTombstone); + } + + await AssertCheckpointImageAsync(verify, scenario.History); + CloudStateCheckpoint? checkpoint = await verify.Checkpoints.GetAsync(CloudLocalChangeFeed.CheckpointName); + Assert.False(checkpoint is not null && LocalChangeCheckpoint.Decode(checkpoint.Value).RequiresFullRescan); + if (!facadeRecovery) + { + Assert.Equal(root.DirectoryIdentity.ItemId, + Assert.Single((await verify.Operations.ListAsync(100)).Where(row => row.Kind == CloudStateOperationKind.Move)).ItemId); + } + } + + private sealed record OrdinaryMoveProvenanceScenario(CloudItemState[] Rows, CloudStateCheckpoint[][] Live, + CloudStateCheckpoint[] History, CloudOperationJournalEntry Pending); +} From 170442fe189138815af410ced8bef34ad6d37f1b Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:40:20 +0800 Subject: [PATCH 39/40] test(items): support checkpoint reads in commit fault fixture --- .../CloudItemCoordinationFailureTests.cs | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/CfSharp.IntegrationTests/CloudItemCoordinationFailureTests.cs b/tests/CfSharp.IntegrationTests/CloudItemCoordinationFailureTests.cs index 1f23cc3..e536014 100644 --- a/tests/CfSharp.IntegrationTests/CloudItemCoordinationFailureTests.cs +++ b/tests/CfSharp.IntegrationTests/CloudItemCoordinationFailureTests.cs @@ -69,7 +69,8 @@ public async Task MoveAndDeletePreserveCompletedFileSystemWorkWhenCommitFails() Assert.Equal("CloudItem.Move", moveFailure.Operation); Assert.Equal(sourcePath, moveFailure.Path); Assert.Equal(movedPath, moveFailure.DestinationPath); - Assert.IsType(moveFailure.InnerException); + IOException moveCommitFailure = Assert.IsType(moveFailure.InnerException); + Assert.Equal("Injected commit failure.", moveCommitFailure.Message); Assert.False(Directory.Exists(sourcePath)); Assert.True( Directory.Exists(movedPath), @@ -83,7 +84,8 @@ public async Task MoveAndDeletePreserveCompletedFileSystemWorkWhenCommitFails() Assert.Equal("CloudItem.Delete", deleteFailure.Operation); Assert.Equal(deletePath, deleteFailure.Path); Assert.Null(deleteFailure.DestinationPath); - Assert.IsType(deleteFailure.InnerException); + IOException deleteCommitFailure = Assert.IsType(deleteFailure.InnerException); + Assert.Equal("Injected commit failure.", deleteCommitFailure.Message); Assert.False(File.Exists(deletePath)); } finally @@ -195,7 +197,7 @@ internal Transaction(IReadOnlyDictionary items) public ICloudItemStateRepository Items => this; - public ICloudCheckpointRepository Checkpoints => throw new NotSupportedException(); + public ICloudCheckpointRepository Checkpoints { get; } = new EmptyCheckpoints(); public ICloudOperationJournal Operations => throw new NotSupportedException(); @@ -293,6 +295,24 @@ public ValueTask RollbackAsync(CancellationToken cancellationToken = default) } public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + private sealed class EmptyCheckpoints : ICloudCheckpointRepository + { + public ValueTask GetAsync(string name, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromResult(null); + } + + // This fault fixture has no captured provenance. Reads must report absence; + // unexpected metadata writes still fail before the deliberate commit fault. + public ValueTask> ListAsync(string namePrefix, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public ValueTask UpsertAsync(CloudStateCheckpoint checkpoint, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + public ValueTask RemoveAsync(string name, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + } } } } From 8ecab286a540b1113530d3955caeeb88ea6fdb86 Mon Sep 17 00:00:00 2001 From: Quaternion8192 <70364461+Quaternion8192@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:48:28 +0800 Subject: [PATCH 40/40] fix(file-system): honor cancellation after move preparation --- docs/placeholders.md | 14 +- .../CloudFileSystem.NamespaceOperations.cs | 1 + .../Items/CloudItem.NamespaceOperations.cs | 5 +- .../CloudDirectoryMoveCancellationTests.cs | 146 ++++++++++++++++++ 4 files changed, 161 insertions(+), 5 deletions(-) create mode 100644 tests/CfSharp.IntegrationTests/CloudDirectoryMoveCancellationTests.cs diff --git a/docs/placeholders.md b/docs/placeholders.md index f816c84..b1438e8 100644 --- a/docs/placeholders.md +++ b/docs/placeholders.md @@ -65,10 +65,16 @@ An approved external Cloud Files rename can change the namespace while a metadat open. Treat the result's native observation as a checked fact at that stage, not a continuing namespace lock; honor a later conflict/rescan even when `DurableProjectionCommitted` is true. -`MoveToAsync` prepares supported managed directories before its native move. Retrying the original -source/destination after a crash uses the retained indexed preparation and performs only verified -projection. To select a specific externally prepared proof, pass `new CloudMoveOptions(proof)`; -it must name that exact target. Missing-source directory retries without evidence fail closed. +`MoveToAsync` prepares supported managed directories before its native move. +Cancellation is checked again after automatic preparation, including capability fallback, +immediately before native movement. Cancellation at that boundary may retain committed proof +metadata while leaving the native source and official item paths unchanged. After native success, +durable recovery completes independently of caller cancellation. + +Retrying the original source/destination after a crash uses the retained indexed preparation and +performs only verified projection. To select a specific externally prepared proof, pass +`new CloudMoveOptions(proof)`; it must name that exact target. Missing-source directory retries +without evidence fail closed. Normal ordinary-directory moves and the existing boolean file-replacement options remain available. Moving an ordinary ancestor relocates any existing managed descendants' live binding paths and rebuilds their mutable membership in the same official subtree transaction. It retains their diff --git a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs index 4bf899f..326271a 100644 --- a/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs +++ b/src/CfSharp/FileSystem/CloudFileSystem.NamespaceOperations.cs @@ -210,6 +210,7 @@ await PersistMovedStateAsync( } } + cancellationToken.ThrowIfCancellationRequested(); try { if (item.Kind is CloudItemKind.File) diff --git a/src/CfSharp/Items/CloudItem.NamespaceOperations.cs b/src/CfSharp/Items/CloudItem.NamespaceOperations.cs index a955155..0e0cc5b 100644 --- a/src/CfSharp/Items/CloudItem.NamespaceOperations.cs +++ b/src/CfSharp/Items/CloudItem.NamespaceOperations.cs @@ -7,7 +7,8 @@ public abstract partial class CloudItem /// One valid destination child name. /// Collision behavior or an original directory proof, or null for no replacement. /// - /// Token observed before the synchronous file-system move and during durable-state work. + /// Token observed during preparation and immediately before the synchronous file-system move, + /// including after capability fallback. Durable recovery after native success ignores cancellation. /// /// /// A new immutable reference and snapshot at the destination. This reference remains bound to @@ -22,6 +23,8 @@ public abstract partial class CloudItem /// durable recovery. Concurrent owner disposal waits for that work; new public operations /// are rejected after stopping begins. Cancellation cannot interrupt durable recovery after /// the synchronous native move succeeds. + /// Cancellation after preparation may leave its metadata committed while the native source + /// and official item paths remain unchanged. /// Known file paths and retained live membership of source and destination ancestors commit /// together, with each ancestor refreshed once. Their native bindings, immutable directory /// preparations and receipts, and existing pending journal operations remain unchanged. diff --git a/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCancellationTests.cs b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCancellationTests.cs new file mode 100644 index 0000000..954a6f8 --- /dev/null +++ b/tests/CfSharp.IntegrationTests/CloudDirectoryMoveCancellationTests.cs @@ -0,0 +1,146 @@ +using System.Runtime.Versioning; + +namespace CfSharp.IntegrationTests; + +[SupportedOSPlatform("windows10.0.16299")] +public sealed partial class CloudDirectoryMoveTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AutomaticPreparationCancellationPreventsNativeMoveAndRetainsProof(bool capabilityFallback) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudOperationJournalEntry pending = await SeedPendingChildAsync(root); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + string destination = Path.Combine(root.RootPath, "Moved"); + CloudItemState[] originalRows; + CloudStateCheckpoint[] originalLive; + await using (ICloudStateTransaction read = await root.Store.BeginTransactionAsync()) + { + originalRows = (await read.Items.ListSubtreeAsync("Docs")).ToArray(); + originalLive = (await read.Checkpoints.ListAsync("cfsharp/namespace/directories")).ToArray(); + } + + using CancellationTokenSource cancellation = new(); + bool reached = false; + root.Faults.AfterCommit = () => + { + root.Faults.AfterCommit = null; + Assert.True(Directory.Exists(source.FullPath)); + Assert.False(Directory.Exists(destination)); + reached = true; + cancellation.Cancel(); + if (capabilityFallback) + { + // Inject the exact capability exception at the preparation completion + // boundary; native IDs and the committed SQLite proof remain real. + throw new CloudDirectoryEvidenceUnavailableException("Simulated post-preparation capability loss."); + } + }; + try + { + OperationCanceledException error = await Assert.ThrowsAnyAsync(() => + source.MoveToAsync(root.FileSystem.Root, "Moved", cancellationToken: cancellation.Token).AsTask()); + Assert.Equal(cancellation.Token, error.CancellationToken); + } + finally + { + root.Faults.AfterCommit = null; + } + + Assert.True(reached); + Assert.True(Directory.Exists(source.FullPath)); + Assert.False(Directory.Exists(destination)); + CloudDirectoryMoveProof proof = (await CloudDirectoryMoveEvidence.ReadIntentAsync(root.Store, "Docs", "Moved", CancellationToken.None))!; + Assert.NotNull(proof); + CloudStateCheckpoint[] preparation; + await using (ICloudStateTransaction verify = await root.Store.BeginTransactionAsync()) + { + foreach (CloudItemState row in originalRows) + { + CloudItemState current = (await verify.Items.GetByItemIdAsync(row.ItemId))!; + Assert.Equal((row.RemoteId, row.RelativePath, row.Kind, row.RemoteRevision, row.LocalFileId, row.IsTombstone, row.UpdatedAt), + (current.RemoteId, current.RelativePath, current.Kind, current.RemoteRevision, current.LocalFileId, current.IsTombstone, current.UpdatedAt)); + } + + await AssertCheckpointImageAsync(verify, originalLive); + preparation = (await verify.Checkpoints.ListAsync($"cfsharp/namespace/preparations/{proof.ProofId:N}")).ToArray(); + Assert.Equal(2, preparation.Length); + Assert.Null(await verify.Checkpoints.GetAsync(CloudDirectoryMoveEvidence.ReceiptName(proof.ProofId))); + await AssertFileMoveJournalImageAsync(verify, pending); + Assert.Single(await verify.Operations.ListAsync(100)); + } + + await root.RestartAsync(); + CloudItemMoveResult retry = await root.FileSystem.GetDirectory("Docs") + .MoveToAsync(root.FileSystem.Root, "Moved", new CloudMoveOptions(proof)); + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, retry.DirectoryReconciliation!.Outcome); + await AssertProjectedChildAsync(root, "Moved", pending); + await using ICloudStateTransaction after = await root.Store.BeginTransactionAsync(); + await AssertCheckpointImageAsync(after, preparation); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CancellationAfterNativeMoveStillCommitsAutomaticOrFallbackProjection(bool capabilityFallback) + { + await using DirectoryMoveTestRoot root = await DirectoryMoveTestRoot.OpenAsync(); + CloudOperationJournalEntry pending = await SeedPendingChildAsync(root); + CloudDirectory source = root.FileSystem.GetDirectory("Docs"); + string destination = Path.Combine(root.RootPath, "Moved"); + using CancellationTokenSource cancellation = new(); + int commitsStarted = 0; + int commitsCompleted = 0; + root.Faults.BeforeCommit = () => + { + if (++commitsStarted == 2) + { + Assert.False(Directory.Exists(source.FullPath)); + Assert.True(Directory.Exists(destination)); + cancellation.Cancel(); + } + }; + root.Faults.AfterCommit = () => + { + if (++commitsCompleted == 1) + { + Assert.True(Directory.Exists(source.FullPath)); + Assert.False(Directory.Exists(destination)); + Assert.False(cancellation.IsCancellationRequested); + if (capabilityFallback) + { + throw new CloudDirectoryEvidenceUnavailableException("Simulated post-preparation capability loss."); + } + } + }; + CloudItemMoveResult result; + try + { + result = await source.MoveToAsync(root.FileSystem.Root, "Moved", cancellationToken: cancellation.Token); + } + finally + { + root.Faults.BeforeCommit = null; + root.Faults.AfterCommit = null; + } + + Assert.True(cancellation.IsCancellationRequested); + Assert.Equal(2, commitsStarted); + Assert.Equal(2, commitsCompleted); + Assert.Equal(2, result.DurableStateEntriesUpdated); + if (capabilityFallback) + { + Assert.Null(result.DirectoryReconciliation); + } + else + { + Assert.Equal(CloudDirectoryMoveReconciliationOutcome.Projected, result.DirectoryReconciliation!.Outcome); + Assert.True(result.DirectoryReconciliation.DurableProjectionCommitted); + } + + await root.RestartAsync(); + await AssertProjectedChildAsync(root, "Moved", pending); + } +}