From 3f55e7bb7a366347884c335f875b60f803c44cd4 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 29 Aug 2026 17:42:45 +0900 Subject: [PATCH 01/11] Reuse C# lexer states during cold extraction --- TESTING_GUIDE.md | 2 + ...al-full-index-csharp-line-state.changed.md | 14 +++++++ .../Symbols/SymbolExtractor.CSharpScanner.cs | 9 +++- .../SymbolExtractor.ExtractionPhases.cs | 31 ++++++++++++-- .../Indexer/Symbols/SymbolExtractor.cs | 7 +++- .../SymbolExtractorCSharpRegexProbeTests.cs | 42 +++++++++++++++++++ 6 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-csharp-line-state.changed.md diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index aa49dc2335..e396a6c508 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -339,6 +339,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result C# query-range generic null-comparison regressions place equality and inequality forms in one indexed source when both assert the same absence of leaked enum references. C# query-range collision forms for basic selection, directional ordering, keyword-named members, and object initializers share one indexed source when they assert the same empty inspect reference bundle. C# inspect brace-range regressions place char-literal, raw-string, and verbatim-string forms in one indexed class and query each following method from that shared fixture. + C# lexer-state reuse coverage keeps multiline comments, verbatim strings, raw strings, and genuine properties in one repeated extractor fixture. It compares every `SymbolRecord` field against the fallback path, requires the production path to reuse the initial per-line state snapshot, and keeps a blocking allocation-reduction assertion. C# generic query-range selectors share simple and tuple type arguments in one fixture, while generic type-pattern coverage shares designation and no-designation forms in another fixture. C# field-type grammar coverage shares static readonly tuple, nullable-tuple, generic-tuple-member, const/plain-field controls, and deconstruction negatives in one extractor fixture. The full-scan regression also seeds the previous C# extractor contract against an unchanged file and verifies that incremental indexing restores the tuple field kind and complete return type. C# switch-expression pattern-variable coverage keeps recursive, declaration, guard, and comment-trivia forms in one extractor fixture when their contract is the set of genuine enum-member references. @@ -1289,6 +1290,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" C# named-argument の coverage は構文の判別を `ReferenceExtractorCSharpTests`、永続化後の query / dependency 動作を `QueryCommandRunnerIssue4833Tests` に保持します。positional argument との混在、並べ替え、attribute、constructor、nested / multiline call、value 式側、named `out` declaration、明示型 lambda / anonymous method、および型付き LINQ range variable の type reference、property subpattern の型、ならびに alias、label、nullable type、ternary に対する負例を維持してください（#4833、回帰 #106 / #122）。 C# の修飾付き一般名 call の coverage は、static BCL、instance、LINQ extension、alias 修飾、current instance、未解決 receiver の各 case を同じ fixture に維持します。extraction が全 row を保持すること、無修飾名による references / callers / callees と hotspot count の既定動作が解決済み evidence を維持しつつ未解決 noise を除外すること、completeness option がその noise を決定的に復元すること、dependency query が identity scope のままであることを検証してください（#4867）。 C# member-read coverage は enum / const pattern、通常の修飾付き定数、static readonly field、static property、cross-file target、callable 名の衝突、真の method invocation を同じ fixture に維持します。連携する extractor / full-scan / `DbReaderTests` fixture で、重複 `call` を伴わない `member_read` 抽出、既定 callers / callees / impact からの除外、明示 compatibility option による復元、legacy `call` row の読み取りを検証してください（#4894）。 + C# lexer-state 再利用 coverage は、複数行 comment、verbatim string、raw string、実 property を1つの反復 extractor fixture に維持します。fallback 経路と全 `SymbolRecord` field を比較し、production 経路が最初の行別 state snapshot を再利用することと allocation 削減 assertion を blocking に固定してください。 Crystal、Groovy、Tcl、Prolog、`ambiguous_pl` の graph fixture では、import、括弧付き call、同一ファイルに限定した保守的な command / predicate call、caller container、keyword の false-positive control を個別に診断可能な状態で維持し、capability test の symbol / reference / graph 広告を extractor fixture と一致させてください（#4746）。 さらに database status test でこれらの言語の古い extractor-version stamp と現行 stamp を固定し、graph 対応前の row が authoritative な graph readiness を報告できないことを検証してください。 database page-attribution coverage では、empty / schema-only、WAL から可視な overflow、database を縮小する WAL、後続 WAL commit 後も connection に固定された read snapshot の各 case、`dbstat` 集約と WAL 検証のキャンセル、large page count の上限付き拒否、再照合、破損 file の拒否、main/WAL/SHM の分離、20 object / 128文字の support-safe 出力上限、明示的な unavailable / not-requested 値、`total_changes()` / `PRAGMA query_only` が不変であることを、連携した `DbReaderTests` fixture で維持してください。 diff --git a/changelog.d/unreleased/+initial-full-index-csharp-line-state.changed.md b/changelog.d/unreleased/+initial-full-index-csharp-line-state.changed.md new file mode 100644 index 0000000000..6fd286f31e --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-csharp-line-state.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs +--- + +## English + +- **Cold C# indexing reuses lexer start states** — The initial symbol scan now carries its per-line C# lexer snapshots into later scope analysis instead of lexing the same file a second time, reducing first-index CPU and allocations without changing extracted symbols. + +## 日本語 + +- **C# の初回 index で lexer 開始状態を再利用** — 最初の symbol 走査で得た行別の C# lexer snapshot を後続の scope 解析へ引き継ぎ、同じ file の二重字句解析を避けることで、抽出結果を変えずに初回 index の CPU 時間と allocation を削減しました。 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs index ede8fcc409..59b7b8040c 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs @@ -3918,12 +3918,16 @@ private static string[] BuildCSharpMatchLines( CSharpRegexProbeCounts? csharpRegexProbeCounts, out int[]?[] collapsedToRaw, out string[] scopeLines, - out bool[] testMethodAttributedDeclarationLines) + out bool[] testMethodAttributedDeclarationLines, + out CSharpLexState[]? lineStartStates) { var matchLines = new string[rawLines.Length]; collapsedToRaw = new int[]?[rawLines.Length]; scopeLines = new string[rawLines.Length]; testMethodAttributedDeclarationLines = new bool[rawLines.Length]; + lineStartStates = applyCSharpRegexProbeOptimizations + ? new CSharpLexState[rawLines.Length] + : null; var csharpLexState = new CSharpLexState(); var testAttributeScanner = new CSharpTestAttributePrefixScanner(); var inLeadingAttributeBlock = false; @@ -3933,6 +3937,9 @@ private static string[] BuildCSharpMatchLines( var activeEnumBodyDepth = 0; for (int lineIndex = 0; lineIndex < rawLines.Length; lineIndex++) { + if (lineStartStates != null) + lineStartStates[lineIndex] = csharpLexState; + var lexedLine = LexCSharpLine(rawLines[lineIndex], csharpLexState); csharpLexState = lexedLine.EndState; // Scope scans need the full C# lexer's literal/comment masking while preserving diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs index 4d600ac29d..e067c4548d 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs @@ -8,7 +8,10 @@ private sealed class PatternScanInputs { private readonly string _lang; private readonly string[] _lines; + private readonly CSharpRegexProbeCounts? _csharpRegexProbeCounts; + private readonly bool _csharpLineStartStatesFromInitialScan; private CSharpLexState[]? _csharpLineStartStates; + private bool _csharpLineStartStateReuseRecorded; private DartClassBodyScope? _dartInsideClassBody; private JavaScriptScopePrivacyFlags[][]? _privateScopeColumns; private CSharpTypeBodyScope? _csharpInsideTypeBody; @@ -31,6 +34,7 @@ public PatternScanInputs( { _lang = lang; _lines = lines; + _csharpRegexProbeCounts = csharpRegexProbeCounts; PythonModulePrefix = lang == "python" ? GetPythonModulePrefix(filePath) : null; @@ -81,6 +85,7 @@ public PatternScanInputs( int[]?[] csharpMatchColumnToRaw = null!; string[]? csharpScopeLines = null; bool[]? csharpTestMethodAttributedDeclarationLines = null; + CSharpLexState[]? csharpLineStartStates = null; CSharpMatchLines = lang == "csharp" ? BuildCSharpMatchLines( lines, @@ -91,8 +96,11 @@ public PatternScanInputs( csharpRegexProbeCounts, out csharpMatchColumnToRaw, out csharpScopeLines, - out csharpTestMethodAttributedDeclarationLines) + out csharpTestMethodAttributedDeclarationLines, + out csharpLineStartStates) : null; + _csharpLineStartStates = csharpLineStartStates; + _csharpLineStartStatesFromInitialScan = csharpLineStartStates != null; CSharpMatchColumnToRaw = csharpMatchColumnToRaw; CSharpScopeLines = csharpScopeLines; CSharpTestMethodAttributedDeclarationLines = csharpTestMethodAttributedDeclarationLines; @@ -148,8 +156,25 @@ public CSharpDeclarationStartScope GetCSharpDeclarationStartScope() => CSharpScopeLines!, GetCSharpInsideTypeBody()); - private CSharpLexState[] BuildCSharpLineStartStates() => - _csharpLineStartStates ??= SymbolExtractor.BuildCSharpLineStartStates(_lines); + private CSharpLexState[] BuildCSharpLineStartStates() + { + if (_csharpLineStartStates != null) + { + if (_csharpLineStartStatesFromInitialScan && !_csharpLineStartStateReuseRecorded) + { + _csharpLineStartStateReuseRecorded = true; + if (_csharpRegexProbeCounts != null) + { + _csharpRegexProbeCounts.LineStartStateReuseCount += + _csharpLineStartStates.Length; + } + } + + return _csharpLineStartStates; + } + + return _csharpLineStartStates = SymbolExtractor.BuildCSharpLineStartStates(_lines); + } private JavaScriptScopePrivacyFlags[][] BuildPrivateScopeColumns() { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index e94a6a91e5..f8f49a29c1 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -363,6 +363,7 @@ private sealed class CSharpRegexProbeCounts public int WrappedModifierMatchInputMaterializationCount { get; set; } public int DeclarationPatternRegexAttemptCount { get; set; } public int PhysicalInputNegativePrefixCacheHitCount { get; set; } + public int LineStartStateReuseCount { get; set; } } internal readonly record struct CSharpRegexProbeMetrics( @@ -378,7 +379,8 @@ internal readonly record struct CSharpRegexProbeMetrics( int WrappedModifierPrefixMaterializationCount, int WrappedModifierMatchInputMaterializationCount, int DeclarationPatternRegexAttemptCount, - int PhysicalInputNegativePrefixCacheHitCount); + int PhysicalInputNegativePrefixCacheHitCount, + int LineStartStateReuseCount); internal static List ExtractForRequiredLiteralGateTesting( long fileId, @@ -457,7 +459,8 @@ internal static List ExtractForCSharpRegexProbeTesting( counts.WrappedModifierPrefixMaterializationCount, counts.WrappedModifierMatchInputMaterializationCount, counts.DeclarationPatternRegexAttemptCount, - counts.PhysicalInputNegativePrefixCacheHitCount); + counts.PhysicalInputNegativePrefixCacheHitCount, + counts.LineStartStateReuseCount); return symbols; } diff --git a/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs b/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs index fa8593a59e..cba431b6fa 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs @@ -235,6 +235,48 @@ internal class Cache{{index}} + $"optimized={optimizedAllocatedBytes:N0}, baseline={baselineAllocatedBytes:N0} bytes."); } + [Fact] + public void Extract_CSharpLineStartStates_ReuseInitialLexerPassAndReduceAllocations() + { + var content = string.Join( + '\n', + Enumerable.Range(0, 32).Select(index => $$"""" + internal class Lexed{{index}} + { + /* A multiline comment with declaration-shaped noise: + public int Hidden { get; set; } + */ + private const string Verbatim = @"first + { still literal }"; + private const string Raw = """ + public int AlsoHidden { get; set; } + """; + public int Value { get; set; } + } + """")); + + var baseline = Extract(content, applyOptimizations: false, out var baselineMetrics); + var optimized = Extract(content, applyOptimizations: true, out var optimizedMetrics); + + AssertSymbolsEqual(baseline, optimized); + Assert.Equal(32, optimized.Count(symbol => symbol.Kind == "class")); + Assert.Equal(32, optimized.Count(symbol => symbol.Kind == "property" && symbol.Name == "Value")); + Assert.DoesNotContain(optimized, symbol => symbol.Name is "Hidden" or "AlsoHidden"); + Assert.Equal(0, baselineMetrics.LineStartStateReuseCount); + Assert.Equal(content.Split('\n').Length, optimizedMetrics.LineStartStateReuseCount); + + _ = Extract(content, applyOptimizations: false, out _); + _ = Extract(content, applyOptimizations: true, out _); + + var baselineAllocatedBytes = MeasureAllocatedBytes(content, applyOptimizations: false); + var optimizedAllocatedBytes = MeasureAllocatedBytes(content, applyOptimizations: true); + + Assert.True( + optimizedAllocatedBytes < baselineAllocatedBytes, + $"Expected initial lexer-state reuse to allocate less: " + + $"optimized={optimizedAllocatedBytes:N0}, baseline={baselineAllocatedBytes:N0} bytes."); + } + [Theory] [InlineData(false)] [InlineData(true)] From 6ab161b0c5f5237d28a85b729d43d355d88a20b4 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 29 Aug 2026 18:01:19 +0900 Subject: [PATCH 02/11] Scale cold C# prepass artifact reuse --- DEVELOPER_GUIDE.md | 11 ++- TESTING_GUIDE.md | 2 + ...full-index-csharp-prepass-cache.changed.md | 14 ++++ .../CSharpPrepassSymbolArtifactCache.cs | 2 +- .../Indexer/CSharpStaticInterfacePrepass.cs | 29 +++++++- .../CSharpPrepassSymbolArtifactCacheTests.cs | 73 +++++++++++++++++++ 6 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-csharp-prepass-cache.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ffdebf84bb..678a2e26ab 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -538,8 +538,11 @@ bounded-regex issue reporting remain on the normal main-pass path. Incomplete prepasses, extraction-stall test seams, checksum drift, regex timeouts, and cache admission limits fall back to ordinary extraction. A timed-out prepass result is partial and must not make that transient result authoritative. Keep admission -bounded to 4,096 files, 131,072 symbols, and an estimated 32 MiB, and clear all -unconsumed artifacts before reference-graph work begins. +bounded to 131,072 symbols and an estimated 32 MiB rather than an independent +production file-count ceiling. When a bound is reached, admit larger decoded +sources first with original candidate order as the deterministic tie-breaker; +build immutable lookups in semantic order before this admission ordering. Clear +all unconsumed artifacts before reference-graph work begins. The workspace qualified-pattern lookup needs only raw non-enum type names for enum-shadowing decisions. Build that conflict set directly; do not call the @@ -4745,7 +4748,9 @@ extraction には main pass と同じ absolute file path / project root を渡 persistence、reference extraction、bounded-regex issue は通常の main-pass 経路で処理してください。 不完全な prepass、extraction-stall test seam、checksum drift、regex timeout、cache 上限では通常 extraction へ fallback します。timeout した prepass 結果は partial であり、一過性の結果を -authoritative にしてはいけません。admission は 4,096 file、131,072 symbol、推定 32 MiB に制限し、未消費 +authoritative にしてはいけません。admission は独立した production file-count 上限ではなく、131,072 symbol と +推定 32 MiB に制限してください。上限へ達する場合は decode 済み source の大きい順、同じ size では元の +candidate 順で admit し、この順序付けより前に immutable lookup を意味上の順序で構築してください。未消費 artifact は reference graph 開始前にすべて clear してください。 workspace qualified-pattern lookup が enum shadowing 判定に必要とするのは raw な non-enum type diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index e396a6c508..3c6c8bb640 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -483,6 +483,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result - `Run_FullScan_PostPrepassCsharpContractLeavesReadinessPartialUntilCleanRetry` keeps the full-scan extraction state monotonic across fresh, rebuild, and incremental-existing routes. Its ordered Python-before-C# fixture must prove that the earlier raw chunk persists and remains searchable through both standard and trigram FTS after the later C# workspace snapshot drifts, with exactly one bulk-load optimization. - `Run_FullScan_FatalParallelResultKeepsWorkerResourcesAliveUntilPeersStop` blocks one C# symbols worker while a peer reports a fatal extraction stall. Keep the command return prompt, assert that neither worker completion nor artifact-cache clearing occurs before the blocked peer is released, and wait for both cleanup signals before restoring process-wide hooks or deleting the fixture. - `CSharpPrepassSymbolArtifactCacheTests`, `FileIndexerTests`, and the CLI/MCP fresh-index fixtures protect bounded prepass artifact reuse. Keep deep-clone independence for generic admission; production owned-list admission must retain list/symbol identity only after successful atomic publication, leave rejected or cancelled input caller-owned, and release workspace fallback symbols only after both lookup snapshots are materialized. Preserve lookup parity and mutation isolation, take-once checksum matching, mismatch consumption, atomic file/symbol/estimated-byte caps, cancellation without partial admission, and non-admission of partial symbols after any bounded-regex timeout. Encoding theories must compare UTF-8, UTF-16 LE/BE, and invalid-UTF-8 prepass checksums with the authoritative loader. Integration coverage must prove reuse only for an empty non-rebuild full index, ordinary extraction for rebuild/symbols-only/existing/incomplete-or-stall paths, authoritative main-read mutation fallback, unchanged post hooks and family/kind processing, and cache clearing before graph work. + The default cache must admit at least 4,097 sparse artifacts while remaining bounded by symbol and estimated-byte budgets. When an injected capacity binds, keep both files in the immutable lookup snapshot but retain the larger decoded source artifact first, using candidate order only as a stable tie-breaker. - `SymbolExtractorRequiredLiteralGateTests` keeps built-in required-literal gating deterministic and output-preserving. It pins 400 audited single-literal Tier A patterns across 51 case-sensitive languages plus six mutually exclusive any-of gates for JavaScript/TypeScript HOCs, both TypeScript @@ -1635,6 +1636,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `Run_FullScan_PostPrepassCsharpContractLeavesReadinessPartialUntilCleanRetry` は fresh、rebuild、incremental-existing の各 route で full-scan extraction state が単調に維持されることを固定します。Python を C# より先に処理する fixture で、後段の C# workspace snapshot drift 後も先行 raw chunk が保存され、standard / trigram FTS の両方から検索でき、bulk-load optimization が厳密に1回であることを証明してください。 - `Run_FullScan_FatalParallelResultKeepsWorkerResourcesAliveUntilPeersStop` は一方の C# symbols worker を block し、peer に fatal extraction stall を返させます。command が速やかに戻ること、block 中の peer を release する前に worker completion と artifact-cache clear のどちらも起きないことを assertion し、process-wide hook の復元や fixture 削除の前に両 cleanup signal を待ってください。 - `CSharpPrepassSymbolArtifactCacheTests`、`FileIndexerTests`、CLI/MCP の fresh-index fixture は bounded prepass artifact reuse を固定します。汎用 admission の deep-clone 独立性を維持し、production の owned-list admission は原子的な publish 成功後だけ list / symbol identity を保持し、reject または cancel された input は caller-owned のままにしてください。2種類の lookup snapshot を materialize した後だけ workspace fallback symbol を解放し、lookup parity と mutation isolation を保ちます。checksum 一致時の take-once、不一致時の消費、file / symbol / estimated-byte cap の原子性、partial admission を残さない cancellation、bounded-regex timeout 後の partial symbol をadmitしない契約も維持してください。encoding theory は UTF-8、UTF-16 LE/BE、不正 UTF-8 の prepass checksum を authoritative loader と比較します。integration coverage では空 database の非 rebuild full index だけが再利用し、rebuild / symbols-only / existing / incomplete-or-stall 経路は通常 extraction、main read 中の mutation は checksum fallback、post hook と family/kind 処理は従来どおり、graph 開始前に cache が clear されることを証明してください。 + 既定 cache は symbol / 推定 byte budget で bounded なまま、少なくとも4,097件の疎な artifact を admit してください。注入した容量上限に達する場合も両 file を immutable lookup snapshot に残し、decode 済み source の大きい artifact を先に保持し、candidate 順は安定した tie-breaker としてだけ使います。 - `CSharpPrepassSymbolArtifactCacheTests.CSharpWorkspaceAssembly_PreservesOrderIdentityAndEvidence` と `PerformanceTests.CSharpPrepassWorkspaceSegments_AvoidFlattenedReferenceBuffers` は、existing row / candidate / file内symbol順を保ったまま一時flatten bufferなしでprepass workspaceを組み立てる契約を固定します。allocation guardはcache上限131,072 symbolを16 KiB未満で全列挙します。2種類のimmutable lookupが完成するまでnon-owning viewを維持し、その後だけowned listを移譲してください。 - `SymbolExtractorRequiredLiteralGateTests` は built-in required-literal gate の決定性と output 不変性を固定します。51 の case-sensitive 言語にまたがる監査済み single-literal Tier A pattern diff --git a/changelog.d/unreleased/+initial-full-index-csharp-prepass-cache.changed.md b/changelog.d/unreleased/+initial-full-index-csharp-prepass-cache.changed.md new file mode 100644 index 0000000000..9654fc525b --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-csharp-prepass-cache.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs + - src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs +--- + +## English + +- **Cold C# prepass caching now scales past 4,096 files** — Fresh full indexes retain reusable symbol artifacts up to the existing symbol and estimated-memory budgets, prioritizing larger decoded sources when capacity binds so costly extraction work is less likely to repeat. + +## 日本語 + +- **C# の初回事前解析 cache が4,096 fileを超えて拡張** — fresh full index では既存の symbol 数・推定 memory budget まで再利用可能な artifact を保持し、容量到達時は decode 済み source の大きい順に優先して、高コストな抽出の繰り返しを減らします。 diff --git a/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs b/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs index 9df0f85dfc..4f4041554f 100644 --- a/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs +++ b/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs @@ -6,7 +6,7 @@ namespace CodeIndex.Indexer; internal sealed class CSharpPrepassSymbolArtifactCache { - internal const int DefaultMaxFiles = 4_096; + internal const int DefaultMaxFiles = int.MaxValue; internal const int DefaultMaxSymbols = 131_072; internal const long DefaultMaxEstimatedBytes = 32L * 1024 * 1024; diff --git a/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs b/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs index 0975d134f7..4d98ed27a6 100644 --- a/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs +++ b/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs @@ -87,6 +87,9 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( var artifactHadRegexTimeouts = symbolArtifactCache == null ? null : new bool[candidates.Count]; + var artifactSourceLengths = symbolArtifactCache == null + ? null + : new int[candidates.Count]; var sourceEvidenceComplete = 1; var hasPendingQualifiedMemberAccessCandidate = 0; string? firstIncompleteSourcePath = null; @@ -165,6 +168,8 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( extractionFilePath, extractionProjectRoot, cancellationToken: cancellationToken); + if (artifactSourceLengths != null) + artifactSourceLengths[candidateIndex] = content.Length; if (regexTimeouts != null) { artifactChecksums![candidateIndex] = checksum; @@ -254,6 +259,7 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( // prepass snapshot を分離する。 if (symbolArtifactCache != null) { + var artifactAdmissions = new List<(int CandidateIndex, int SourceLength)>(); for (var candidateIndex = 0; candidateIndex < extractedByCandidate.Length; candidateIndex++) @@ -263,9 +269,30 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( if (extracted == null || checksum == null) continue; + artifactAdmissions.Add( + (candidateIndex, artifactSourceLengths![candidateIndex])); + } + + // When a retained-artifact budget binds, prefer the decoded sources that + // would be most expensive to extract again. Candidate order remains the + // deterministic tie-breaker, and lookup construction above retains the + // original semantic order. + // artifact budget 到達時は再抽出 cost の大きい source を優先し、同じ + // size では元の candidate 順を維持する。lookup の意味順は上で確定済み。 + artifactAdmissions.Sort(static (left, right) => + { + var lengthComparison = right.SourceLength.CompareTo(left.SourceLength); + return lengthComparison != 0 + ? lengthComparison + : left.CandidateIndex.CompareTo(right.CandidateIndex); + }); + foreach (var admission in artifactAdmissions) + { + var candidateIndex = admission.CandidateIndex; + var extracted = extractedByCandidate[candidateIndex]!; symbolArtifactCache.TryAdmitOwned( candidates[candidateIndex].IndexPath, - checksum, + artifactChecksums![candidateIndex]!, extracted, artifactHadRegexTimeouts![candidateIndex], cancellationToken); diff --git a/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs b/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs index 0e4d37efd5..7a89441c13 100644 --- a/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs +++ b/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs @@ -408,6 +408,79 @@ public void TryAdmit_EnforcesFileSymbolAndEstimatedByteCapsWithoutPartialPublish Assert.Equal(0, byteBound.AdmittedEstimatedBytes); } + [Fact] + public void TryAdmit_DefaultCapacityRetainsArtifactsBeyondLegacyFileCeiling() + { + const int artifactCount = 4_097; + var cache = new CSharpPrepassSymbolArtifactCache(); + + for (var index = 0; index < artifactCount; index++) + { + Assert.True(cache.TryAdmit( + $"src/Artifact{index}.cs", + "checksum", + Array.Empty(), + hadRegexTimeout: false)); + } + + Assert.Equal(artifactCount, cache.AdmittedFileCount); + Assert.True(cache.AdmittedEstimatedBytes < CSharpPrepassSymbolArtifactCache.DefaultMaxEstimatedBytes); + } + + [Fact] + public void BuildWorkspaceSymbols_WhenCapacityBindsRetainsLargestSourceWithoutChangingLookups() + { + using var project = TestProjectHelper.CreateTempProjectScope( + "csharp_prepass_artifact_priority"); + var smallPath = TestProjectHelper.WriteTextFile( + project.Root, + "src/Small.cs", + "public interface ISmall { static abstract int Create(); }"); + var largePath = TestProjectHelper.WriteTextFile( + project.Root, + "src/Large.cs", + $$""" + public interface ILarge + { + /* {{new string('x', 8_192)}} */ + static abstract int Create(); + } + """); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + var indexer = new FileIndexer(project.Root, ignoreCase: false); + var smallTarget = CSharpStaticInterfacePrepass.FileTarget.Create( + project.Root, + smallPath, + "csharp"); + var largeTarget = CSharpStaticInterfacePrepass.FileTarget.Create( + project.Root, + largePath, + "csharp"); + var cache = new CSharpPrepassSymbolArtifactCache( + maxFiles: 1, + maxSymbols: 100, + maxEstimatedBytes: 1_000_000); + + var workspace = CSharpStaticInterfacePrepass.BuildWorkspaceSymbols( + writer, + indexer, + [smallTarget, largeTarget], + includeExistingSymbols: false, + symbolArtifactCache: cache); + + var lookups = FlattenStaticInterfaceLookups(workspace.StaticInterfaceMemberLookups!); + Assert.Contains(lookups, item => item.StartsWith("ISmall:Create:", StringComparison.Ordinal)); + Assert.Contains(lookups, item => item.StartsWith("ILarge:Create:", StringComparison.Ordinal)); + var smallChecksum = indexer.BuildRecord(smallPath).record.Checksum; + var largeChecksum = indexer.BuildRecord(largePath).record.Checksum; + Assert.NotNull(smallChecksum); + Assert.NotNull(largeChecksum); + Assert.False(cache.TryTake(smallTarget.IndexPath, smallChecksum, out _)); + Assert.True(cache.TryTake(largeTarget.IndexPath, largeChecksum, out _)); + } + [Fact] public void TryAdmit_CancellationDoesNotPublishPartialArtifact() { From ada36502d5553fdec5af9133ffb8d63726216360 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 29 Aug 2026 18:08:04 +0900 Subject: [PATCH 03/11] Reduce repeated C# property probes --- TESTING_GUIDE.md | 2 + ...ll-index-csharp-property-probes.changed.md | 14 +++++ .../Symbols/SymbolExtractor.CSharpScanner.cs | 26 ++++++++++ .../Symbols/SymbolExtractor.PatternFlow.cs | 5 ++ .../Symbols/SymbolExtractor.PatternLoop.cs | 37 ++++++++++---- .../Indexer/Symbols/SymbolExtractor.cs | 6 +++ .../SymbolExtractorCSharpRegexProbeTests.cs | 51 +++++++++++++++++++ 7 files changed, 130 insertions(+), 11 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-csharp-property-probes.changed.md diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 3c6c8bb640..03bccd62ec 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -340,6 +340,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result C# query-range collision forms for basic selection, directional ordering, keyword-named members, and object initializers share one indexed source when they assert the same empty inspect reference bundle. C# inspect brace-range regressions place char-literal, raw-string, and verbatim-string forms in one indexed class and query each following method from that shared fixture. C# lexer-state reuse coverage keeps multiline comments, verbatim strings, raw strings, and genuine properties in one repeated extractor fixture. It compares every `SymbolRecord` field against the fallback path, requires the production path to reuse the initial per-line state snapshot, and keeps a blocking allocation-reduction assertion. + C# property-probe coverage keeps same-line sibling restarts, tuple-return properties, Microsoft-style brace properties, method bodies, structural braces, and preprocessor lines in shared fixtures. It requires one multiline-candidate build per physical line in the optimized path, allocation-free structural rejection, complete `SymbolRecord` parity, fewer header-regex attempts, and lower allocations than the fallback path. C# generic query-range selectors share simple and tuple type arguments in one fixture, while generic type-pattern coverage shares designation and no-designation forms in another fixture. C# field-type grammar coverage shares static readonly tuple, nullable-tuple, generic-tuple-member, const/plain-field controls, and deconstruction negatives in one extractor fixture. The full-scan regression also seeds the previous C# extractor contract against an unchanged file and verifies that incremental indexing restores the tuple field kind and complete return type. C# switch-expression pattern-variable coverage keeps recursive, declaration, guard, and comment-trivia forms in one extractor fixture when their contract is the set of genuine enum-member references. @@ -1292,6 +1293,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" C# の修飾付き一般名 call の coverage は、static BCL、instance、LINQ extension、alias 修飾、current instance、未解決 receiver の各 case を同じ fixture に維持します。extraction が全 row を保持すること、無修飾名による references / callers / callees と hotspot count の既定動作が解決済み evidence を維持しつつ未解決 noise を除外すること、completeness option がその noise を決定的に復元すること、dependency query が identity scope のままであることを検証してください（#4867）。 C# member-read coverage は enum / const pattern、通常の修飾付き定数、static readonly field、static property、cross-file target、callable 名の衝突、真の method invocation を同じ fixture に維持します。連携する extractor / full-scan / `DbReaderTests` fixture で、重複 `call` を伴わない `member_read` 抽出、既定 callers / callees / impact からの除外、明示 compatibility option による復元、legacy `call` row の読み取りを検証してください（#4894）。 C# lexer-state 再利用 coverage は、複数行 comment、verbatim string、raw string、実 property を1つの反復 extractor fixture に維持します。fallback 経路と全 `SymbolRecord` field を比較し、production 経路が最初の行別 state snapshot を再利用することと allocation 削減 assertion を blocking に固定してください。 + C# property-probe coverage は、同一行 sibling の再開、tuple 戻り値 property、Microsoft-style brace property、method body、構造 brace、preprocessor 行を共有 fixture に維持します。最適化経路では物理行ごとに複数行 candidate を1回だけ構築し、allocation なしの構造 reject、全 `SymbolRecord` の一致、header regex 試行と allocation の削減を固定してください。 Crystal、Groovy、Tcl、Prolog、`ambiguous_pl` の graph fixture では、import、括弧付き call、同一ファイルに限定した保守的な command / predicate call、caller container、keyword の false-positive control を個別に診断可能な状態で維持し、capability test の symbol / reference / graph 広告を extractor fixture と一致させてください（#4746）。 さらに database status test でこれらの言語の古い extractor-version stamp と現行 stamp を固定し、graph 対応前の row が authoritative な graph readiness を報告できないことを検証してください。 database page-attribution coverage では、empty / schema-only、WAL から可視な overflow、database を縮小する WAL、後続 WAL commit 後も connection に固定された read snapshot の各 case、`dbstat` 集約と WAL 検証のキャンセル、large page count の上限付き拒否、再照合、破損 file の拒否、main/WAL/SHM の分離、20 object / 128文字の support-safe 出力上限、明示的な unavailable / not-requested 値、`total_changes()` / `PRAGMA query_only` が不変であることを、連携した `DbReaderTests` fixture で維持してください。 diff --git a/changelog.d/unreleased/+initial-full-index-csharp-property-probes.changed.md b/changelog.d/unreleased/+initial-full-index-csharp-property-probes.changed.md new file mode 100644 index 0000000000..f2f1506f4e --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-csharp-property-probes.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.PatternLoop.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs +--- + +## English + +- **Cold C# indexing avoids repeated property probes** — Same-line declaration restarts now share one multiline property candidate, while completed method and structural lines bypass unnecessary property-header regexes without changing extracted symbols. + +## 日本語 + +- **C# の初回 index で property probe の重複を回避** — 同一行の宣言再開で複数行 property candidate を共有し、完結した method・構造行では不要な property-header regex を省いて、抽出結果を変えずに処理量を削減しました。 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs index 59b7b8040c..5237b61928 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs @@ -1560,6 +1560,8 @@ private static CSharpPropertyMatchCandidate BuildCSharpPropertyMatchLine( bool applyCSharpRegexProbeOptimizations, CSharpRegexProbeCounts? csharpRegexProbeCounts) { + if (csharpRegexProbeCounts != null) + csharpRegexProbeCounts.PropertyCandidateBuildCount++; var matchLine = csharpMatchLines[startLineIndex]; var matchLineSpan = matchLine.AsSpan(); if (IsCSharpNonMemberHeaderLine(matchLineSpan)) @@ -1573,6 +1575,14 @@ private static CSharpPropertyMatchCandidate BuildCSharpPropertyMatchLine( return new CSharpPropertyMatchCandidate(matchLine, startLineIndex, startLineIndex); } + if (applyCSharpRegexProbeOptimizations + && CanSkipCSharpPropertyStructuralShape(trimmedMatchLine)) + { + if (csharpRegexProbeCounts != null) + csharpRegexProbeCounts.PropertyStructuralShapeSkipCount++; + return new CSharpPropertyMatchCandidate(matchLine, startLineIndex, startLineIndex); + } + if (IsCSharpDeclarationExpressionArrow(matchLine) && TryFindCSharpExpressionArrow(lines, startLineIndex, startLineIndex, out var sameLineArrowLineIndex, out var sameLineArrowColumn)) { @@ -1762,6 +1772,22 @@ private static bool CanSkipCSharpPropertyPrefixRegexes(ReadOnlySpan line) || terminal == '=' && line.IndexOf('(') < 0; } + private static bool CanSkipCSharpPropertyStructuralShape(ReadOnlySpan line) + { + if (line[0] is '{' or '}' or '#') + return true; + if (line[^1] != '{') + return false; + + for (var index = line.Length - 2; index >= 0; index--) + { + if (!char.IsWhiteSpace(line[index])) + return line[index] == ')'; + } + + return true; + } + private static bool IsCSharpNonMemberHeaderLine(ReadOnlySpan line) { var trimmed = line.TrimStart(); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.PatternFlow.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.PatternFlow.cs index 42c347e186..0d4f73edd4 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.PatternFlow.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.PatternFlow.cs @@ -33,6 +33,11 @@ private readonly record struct PatternLineScanContext( int LineIndex, PreparedPatternLine PreparedLine); + private struct PatternLineScanState + { + public CSharpPropertyMatchCandidate? CSharpPropertyCandidateForLine; + } + // Candidate-local proof cache shared only by the outer C# pattern loop and its // recoverable-pattern helper. The count encoding keeps default(struct) empty and prevents // an out-of-order miss from claiming an unproven gap in pattern priority. diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.PatternLoop.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.PatternLoop.cs index 5318d4629c..d73f8a942d 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.PatternLoop.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.PatternLoop.cs @@ -55,11 +55,15 @@ private static void ScanPreparedPatternLine( PreparedPatternLine preparedLine) { var lineContext = new PatternLineScanContext(context, i, preparedLine); + var lineScanState = new PatternLineScanState(); var patternStartOffset = preparedLine.PatternStartOffset; while (patternStartOffset >= 0 && patternStartOffset < preparedLine.MatchLine.Length) { - var result = ScanPatternListAtOffset(lineContext, patternStartOffset); + var result = ScanPatternListAtOffset( + lineContext, + patternStartOffset, + ref lineScanState); if (result.Flow != PatternScanFlow.RestartPatternList || result.NextOffset <= patternStartOffset) { @@ -72,7 +76,8 @@ private static void ScanPreparedPatternLine( private static PatternScanResult ScanPatternListAtOffset( PatternLineScanContext lineContext, - int patternStartOffset) + int patternStartOffset, + ref PatternLineScanState lineScanState) { var extraction = lineContext.Extraction; var patternStartState = new PatternStartScanState(); @@ -87,7 +92,8 @@ private static PatternScanResult ScanPatternListAtOffset( pattern, patternIndex, patternStartOffset, - ref patternStartState); + ref patternStartState, + ref lineScanState); var result = ScanApplicablePattern( ref patternScan, ref patternStartState); @@ -132,7 +138,8 @@ private static PatternCandidateScan CreatePatternCandidateScan( SymbolPattern pattern, int patternIndex, int patternStartOffset, - ref PatternStartScanState patternStartState) + ref PatternStartScanState patternStartState, + ref PatternLineScanState lineScanState) { var extraction = lineContext.Extraction; var lines = extraction.Lines; @@ -152,13 +159,21 @@ private static PatternCandidateScan CreatePatternCandidateScan( // 受け付けないため影響を受けず、merger は元の行をそのまま返す。Closes #355. var csharpPropertyCandidate = extraction.Lang == "csharp" && pattern.Kind is "property" or "function" - ? patternStartState.CSharpPropertyCandidateForLine ??= - BuildCSharpPropertyMatchLine( - lines, - csharpMatchLines!, - i, - extraction.ApplyCSharpRegexProbeOptimizations, - extraction.CSharpRegexProbeCounts) + ? extraction.ApplyCSharpRegexProbeOptimizations + ? lineScanState.CSharpPropertyCandidateForLine ??= + BuildCSharpPropertyMatchLine( + lines, + csharpMatchLines!, + i, + applyCSharpRegexProbeOptimizations: true, + extraction.CSharpRegexProbeCounts) + : patternStartState.CSharpPropertyCandidateForLine ??= + BuildCSharpPropertyMatchLine( + lines, + csharpMatchLines!, + i, + applyCSharpRegexProbeOptimizations: false, + extraction.CSharpRegexProbeCounts) : new CSharpPropertyMatchCandidate( lineContext.PreparedLine.MatchLine, i, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index f8f49a29c1..61f1592c77 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -351,6 +351,8 @@ private sealed class RequiredLiteralGateCounts private sealed class CSharpRegexProbeCounts { public int PropertyPrefixSuffixSkipCount { get; set; } + public int PropertyCandidateBuildCount { get; set; } + public int PropertyStructuralShapeSkipCount { get; set; } public int PropertyHeaderRegexAttemptCount { get; set; } public int MethodHeaderRegexAttemptCount { get; set; } public int PlainFieldTerminatorSkipCount { get; set; } @@ -368,6 +370,8 @@ private sealed class CSharpRegexProbeCounts internal readonly record struct CSharpRegexProbeMetrics( int PropertyPrefixSuffixSkipCount, + int PropertyCandidateBuildCount, + int PropertyStructuralShapeSkipCount, int PropertyHeaderRegexAttemptCount, int MethodHeaderRegexAttemptCount, int PlainFieldTerminatorSkipCount, @@ -448,6 +452,8 @@ internal static List ExtractForCSharpRegexProbeTesting( csharpRegexProbeCounts: counts); metrics = new CSharpRegexProbeMetrics( counts.PropertyPrefixSuffixSkipCount, + counts.PropertyCandidateBuildCount, + counts.PropertyStructuralShapeSkipCount, counts.PropertyHeaderRegexAttemptCount, counts.MethodHeaderRegexAttemptCount, counts.PlainFieldTerminatorSkipCount, diff --git a/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs b/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs index cba431b6fa..03ebc8584a 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorCSharpRegexProbeTests.cs @@ -53,6 +53,9 @@ internal class Inline { public int SameLine; public void Run() { } } Assert.True( optimizedMetrics.DeclarationPatternRegexAttemptCount < baselineMetrics.DeclarationPatternRegexAttemptCount); + Assert.True( + optimizedMetrics.PropertyCandidateBuildCount + < baselineMetrics.PropertyCandidateBuildCount); } [Fact] @@ -277,6 +280,54 @@ internal class Lexed{{index}} + $"optimized={optimizedAllocatedBytes:N0}, baseline={baselineAllocatedBytes:N0} bytes."); } + [Fact] + public void Extract_CSharpPropertyStructuralGate_PreservesPropertyFormsAndReducesAllocations() + { + var content = string.Join( + '\n', + Enumerable.Range(0, 48).Select(index => $$""" + #region Type{{index}} + internal class Shape{{index}} + { + public (int Left, int Right) Pair + { + get; + } + + public int Wrap { + get; + } + + public void Run(int value) { + } + } + #endregion + """)); + + var baseline = Extract(content, applyOptimizations: false, out var baselineMetrics); + var optimized = Extract(content, applyOptimizations: true, out var optimizedMetrics); + + AssertSymbolsEqual(baseline, optimized); + Assert.Equal(48, optimized.Count(symbol => symbol.Kind == "property" && symbol.Name == "Pair")); + Assert.Equal(48, optimized.Count(symbol => symbol.Kind == "property" && symbol.Name == "Wrap")); + Assert.Equal(48, optimized.Count(symbol => symbol.Kind == "function" && symbol.Name == "Run")); + Assert.Equal(0, baselineMetrics.PropertyStructuralShapeSkipCount); + Assert.True(optimizedMetrics.PropertyStructuralShapeSkipCount > 0); + Assert.True( + optimizedMetrics.PropertyHeaderRegexAttemptCount + < baselineMetrics.PropertyHeaderRegexAttemptCount); + + _ = Extract(content, applyOptimizations: false, out _); + _ = Extract(content, applyOptimizations: true, out _); + var baselineAllocatedBytes = MeasureAllocatedBytes(content, applyOptimizations: false); + var optimizedAllocatedBytes = MeasureAllocatedBytes(content, applyOptimizations: true); + + Assert.True( + optimizedAllocatedBytes < baselineAllocatedBytes, + $"Expected property structural gating to allocate less: " + + $"optimized={optimizedAllocatedBytes:N0}, baseline={baselineAllocatedBytes:N0} bytes."); + } + [Theory] [InlineData(false)] [InlineData(true)] From d76e4bbcf96be136f72c7db82db6e43485ab94d3 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 29 Aug 2026 18:54:21 +0900 Subject: [PATCH 04/11] Stream symbol worker request content --- DEVELOPER_GUIDE.md | 4 +- TESTING_GUIDE.md | 8 +- ...l-index-symbol-worker-streaming.changed.md | 13 ++ .../Indexer/Symbols/SymbolExtractionWorker.cs | 136 +++++++++++++++++- .../IndexCommandRunnerTests.cs | 108 +++++++++++++- 5 files changed, 253 insertions(+), 16 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-symbol-worker-streaming.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 678a2e26ab..3c35842a2c 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -276,7 +276,7 @@ Set `CDIDX_SLOW_QUERY_MS=` to write slow SQLite command diagnostic | Path | Contract | |---|---| -| Worker protocol JSON | Isolated worker stdin frames are read through `BoundedLineReader`. The symbol-worker client serializes requests directly to UTF-8 and writes newline-framed bytes to the process stream; the worker serializes responses directly to its stdout stream, and the client reads each bounded response frame as UTF-8 bytes for direct deserialization. This avoids an additional UTF-16 JSON string and encoding pass in each direction for every source file. The default frame cap is 32 MiB for both characters and UTF-8 bytes. When a larger `--max-file-bytes` setting needs JSON-escaping headroom, the protocol frame cap may expand up to `WorkerProtocolLineLimits.MaxExtendedLineUtf8Bytes` (384 MiB), never to `int.MaxValue`. `WorkerProtocolJsonValidator` rejects payloads over the negotiated character/UTF-8 byte cap before `JsonDocument.Parse`, parses with `DefaultMaxJsonDepth` (32), rejects more than 1,000,000 object properties, and rejects strings longer than the frame cap. | +| Worker protocol JSON | Isolated worker stdin frames are read through `BoundedLineReader`. The symbol-worker client serializes small request metadata directly to UTF-8, then JSON-escapes source content through fixed-size pooled buffers while writing the newline-framed request to the process stream; it must not retain a source-sized JSON byte array. The writer counts emitted bytes and rejects the frame before exceeding its negotiated cap. The worker serializes responses directly to stdout, and the client reads each bounded response frame as UTF-8 bytes for direct deserialization. This avoids additional source-sized JSON strings and encoding buffers in each direction for every source file. The default frame cap is 32 MiB for both characters and UTF-8 bytes. When a larger `--max-file-bytes` setting needs JSON-escaping headroom, the protocol frame cap may expand up to `WorkerProtocolLineLimits.MaxExtendedLineUtf8Bytes` (384 MiB), never to `int.MaxValue`. `WorkerProtocolJsonValidator` rejects payloads over the negotiated character/UTF-8 byte cap before `JsonDocument.Parse`, parses with `DefaultMaxJsonDepth` (32), rejects more than 1,000,000 object properties, and rejects strings longer than the frame cap. | | User regex find | `find --regex` keeps the classic .NET regex engine for lookaround/backreference compatibility, adds `RegexOptions.CultureInvariant`, adds `IgnoreCase` unless `--exact` is set, and uses `BoundedRegex.DefaultMatchTimeout` per match. Timeouts surface as `E014_REGEX_MATCH_TIMEOUT` / `regex_timeout` in CLI JSON, and human output includes the same recovery hint. `find --all` additionally applies candidate-file and line-scan caps before walking the whole index. | | Shared regex construction | Production regex construction is centralized through `BoundedRegex`, `RegexRegistry`, or `RegexTimeoutPolicy`. Use `BoundedRegex` for extractor patterns and bounded static regex APIs, `RegexRegistry` for raw BCL regex factories that must preserve timeout exceptions (`find --regex`, ignore glob regexes, generated-code path patterns), and `RegexTimeoutPolicy` for diagnostic/redaction surfaces. `RegexRegistry` owns the named ignore-glob timeout (100 ms), generated-code pattern timeout (50 ms), and find-regex factory using `BoundedRegex.DefaultMatchTimeout`. Search-audit recipes treat only `BoundedRegex` aliases and `RegexRegistry.cs` as centralized positive evidence, so new production raw constructors require a deliberate factory or generated-regex entry plus tests. | | Filesystem traversal helpers | `FileSystemTraversalPolicy` keeps top-directory-only enumeration explicit (`IgnoreInaccessible=false`, no implicit recursion) and exposes opt-in `CancellationToken` / entry-budget options. Expected traversal failures are classified centrally so command diagnostics share the same permission, I/O, invalid-path, unsupported-path, path-too-long, and budget-exceeded taxonomy. Existing-child case probes retain one exact-name set capped by `CaseSensitivityProbeDirectory.MaxExistingChildProbeEntries` (4,096), return unknown on truncation so callers use the isolated-write or cached root-policy fallback, and propagate available cancellation tokens. | @@ -4528,7 +4528,7 @@ query コマンドも JSON profile block 用の `--profile` と command-scoped p | 経路 | 契約 | |---|---| -| worker protocol JSON | isolated worker の stdin frame は `BoundedLineReader` で読みます。symbol-worker client は request を直接 UTF-8 に serialize して改行区切りの byte を process stream へ書き、worker は response を stdout stream へ直接 serialize し、client は bounded response frame を UTF-8 byte のまま読み取って直接 deserialize します。これにより source file ごとに両方向で発生していた追加 UTF-16 JSON string と encoding pass を避けます。既定の frame 上限は文字数・UTF-8 byte 数ともに 32 MiB です。大きな `--max-file-bytes` によって JSON escape 分の余裕が必要な場合、protocol frame 上限は `WorkerProtocolLineLimits.MaxExtendedLineUtf8Bytes`（384 MiB）まで拡張できますが、`int.MaxValue` までは拡張しません。`WorkerProtocolJsonValidator` は `JsonDocument.Parse` の前に合意済みの文字数 / UTF-8 byte 上限を超える payload を拒否し、`DefaultMaxJsonDepth`（32）で parse し、object property 1,000,000 件超と frame 上限を超える string を拒否します。 | +| worker protocol JSON | isolated worker の stdin frame は `BoundedLineReader` で読みます。symbol-worker client は小さい request metadata を直接 UTF-8 に serialize し、source content を固定長の pooled buffer で JSON escape しながら改行区切り request を process stream へ書き、source 規模の JSON byte array を保持しません。writer は出力 byte 数を数え、合意済み上限を超える前に frame を拒否します。worker は response を stdout stream へ直接 serialize し、client は bounded response frame を UTF-8 byte のまま読み取って直接 deserialize します。これにより source file ごとに両方向で発生する追加の source 規模 JSON string / encoding buffer を避けます。既定の frame 上限は文字数・UTF-8 byte 数ともに 32 MiB です。大きな `--max-file-bytes` によって JSON escape 分の余裕が必要な場合、protocol frame 上限は `WorkerProtocolLineLimits.MaxExtendedLineUtf8Bytes`（384 MiB）まで拡張できますが、`int.MaxValue` までは拡張しません。`WorkerProtocolJsonValidator` は `JsonDocument.Parse` の前に合意済みの文字数 / UTF-8 byte 上限を超える payload を拒否し、`DefaultMaxJsonDepth`（32）で parse し、object property 1,000,000 件超と frame 上限を超える string を拒否します。 | | user regex find | `find --regex` は lookaround / backreference 互換性のため classic .NET regex engine を維持し、`RegexOptions.CultureInvariant` を付け、`--exact` でない場合は `IgnoreCase` も付け、各 match に `BoundedRegex.DefaultMatchTimeout` を使います。timeout は CLI JSON で `E014_REGEX_MATCH_TIMEOUT` / `regex_timeout` として返り、人間向け出力にも同じ recovery hint が出ます。`find --all` は index 全体を走査する前に candidate file と line scan の上限も適用します。 | | shared regex construction | production の regex 構築は `BoundedRegex`、`RegexRegistry`、または `RegexTimeoutPolicy` に集約します。extractor pattern と bounded static regex API には `BoundedRegex`、timeout 例外を維持する必要がある raw BCL regex factory（`find --regex`、ignore glob regex、generated-code path pattern）には `RegexRegistry`、diagnostic / redaction surface には `RegexTimeoutPolicy` を使います。`RegexRegistry` は ignore glob timeout（100 ms）、generated-code pattern timeout（50 ms）、および `BoundedRegex.DefaultMatchTimeout` を使う find-regex factory の名前付き policy を所有します。search-audit recipe は `BoundedRegex` alias と `RegexRegistry.cs` だけを集約済みの positive evidence と見なすため、新しい production raw constructor は明示的な factory または generated-regex entry とテストを伴う必要があります。 | | filesystem traversal helper | `FileSystemTraversalPolicy` は top-directory-only enumeration を明示し（`IgnoreInaccessible=false`、暗黙の再帰なし）、任意指定の `CancellationToken` / entry budget option を公開します。想定内の traversal failure は中央で分類し、command diagnostic が permission、I/O、invalid-path、unsupported-path、path-too-long、budget-exceeded の taxonomy を共有します。既存 child の case probe は `CaseSensitivityProbeDirectory.MaxExistingChildProbeEntries`（4,096）を上限とする1つの exact-name set だけを保持し、truncation 時は unknown を返して caller の isolated-write または cached root-policy fallback に委ね、利用可能な cancellation token を伝播します。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 03bccd62ec..b503ce0178 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -474,7 +474,9 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result - `IndexCommandRunnerTests.SymbolExtractionWorker_LegacyEnvironmentHooksAreIgnored_Issue3398` launches the isolated symbol worker to prove legacy worker environment variables are ignored. Its callback budget includes process startup and is intentionally wider than ordinary in-process checks so local process load does not turn the legacy-env regression check into a timeout flake (#3863). - `IndexCommandRunnerTests.SymbolExtractionWorker_Utf8RequestsPreserveUnicodeAcrossLanguages` - reuses one isolated worker for C#, Java, TypeScript, Python, Go, and Rust requests whose content and paths contain Japanese text. It first sends an ASCII warm-up with the wider test-only startup budget so parallel net8.0/net9.0 process-startup contention is separated from callback timing, then keeps every Unicode request on the ordinary five-second callback budget. This preserves language-neutral direct UTF-8 request framing and Unicode fidelity without weakening the production worker timeout contract or creating a large fixture (#4937). + reuses one isolated worker for C#, Java, TypeScript, Python, Go, and Rust requests whose content and paths contain Japanese text plus quotes, backslashes, tabs, and mixed line endings. It first sends an ASCII warm-up with the wider test-only startup budget so parallel net8.0/net9.0 process-startup contention is separated from callback timing, then keeps every Unicode request on the ordinary five-second callback budget. This preserves language-neutral direct UTF-8 request framing, JSON escaping, and Unicode fidelity without weakening the production worker timeout contract or creating a large fixture (#4937). +- `IndexCommandRunnerTests.SymbolExtractionWorker_RequestStreamingPreservesJsonAndBoundsAllocation` + round-trips every request field through one BOM-less newline frame, then sends a 512 KiB escape-heavy source to a synchronous non-retaining stream. Keep current-thread allocation below 128 KiB, individual writes at or below 16 KiB, and low-cap failure before the stream receives a byte beyond the negotiated limit. - `IndexCommandRunnerTests.SymbolExtractionWorker_StreamResponseWritesBomlessUtf8Frame` exercises the production stream-response overload with a Japanese C# symbol and verifies one BOM-less, newline-terminated UTF-8 JSON frame. Keep the `StringWriter` protocol tests as the in-process diagnostic path while this test protects process stdout framing. - `BoundedLineReaderTests.ReadUtf8LineAsync_BuffersFramesWithoutDecoding`, `ReadUtf8LineAsync_EnforcesByteLimitBeforeGrowth`, and `ReadUtf8LineAsync_HandlesCrLfAcrossBufferBoundary` @@ -1628,7 +1630,9 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `IndexCommandRunnerTests.SymbolExtractionWorker_LegacyEnvironmentHooksAreIgnored_Issue3398` isolated symbol worker を起動し、legacy worker 環境変数が無視されることを検証します。この callback budget はプロセス起動時間も含むため、通常の in-process チェックより意図的に広く取り、ローカル負荷で legacy-env 回帰テストが timeout flake にならないようにします（#3863）。 - `IndexCommandRunnerTests.SymbolExtractionWorker_Utf8RequestsPreserveUnicodeAcrossLanguages` - 1つの isolated worker を再利用し、日本語の content と path を含む C#、Java、TypeScript、Python、Go、Rust の request を順に送ります。最初に test 専用の広い startup budget で ASCII warm-up を行い、net8.0/net9.0 の並列 process 起動競合と callback の計測を分離したうえで、すべての Unicode request を通常の5秒 callback budget で検証します。production worker の timeout 契約を緩めず、大規模 fixture を作ることなく direct UTF-8 request framing の言語非依存性と Unicode fidelity を固定します（#4937）。 + 1つの isolated worker を再利用し、日本語に加えて quote、backslash、tab、混在改行を持つ content と日本語 path を含む C#、Java、TypeScript、Python、Go、Rust の request を順に送ります。最初に test 専用の広い startup budget で ASCII warm-up を行い、net8.0/net9.0 の並列 process 起動競合と callback の計測を分離したうえで、すべての Unicode request を通常の5秒 callback budget で検証します。production worker の timeout 契約を緩めず、大規模 fixture を作ることなく direct UTF-8 request framing、JSON escape、Unicode fidelity の言語非依存性を固定します（#4937）。 +- `IndexCommandRunnerTests.SymbolExtractionWorker_RequestStreamingPreservesJsonAndBoundsAllocation` + request の全 field を BOM なし改行 frame 1件で round trip し、512 KiB の escape-heavy source を同期完了する非保持 stream へ送ります。current-thread allocation を128 KiB未満、個別 write を16 KiB以下に保ち、低い cap では合意済み上限を1 byteも超えて stream へ書く前に失敗させてください。 - `IndexCommandRunnerTests.SymbolExtractionWorker_StreamResponseWritesBomlessUtf8Frame` 日本語の C# symbol で本番用 stream-response overload を実行し、BOM なし・改行終端の UTF-8 JSON frame が1件出ることを検証します。`StringWriter` の protocol tests は in-process diagnostic 経路として維持し、このテストで process stdout framing を固定します。 - `BoundedLineReaderTests.ReadUtf8LineAsync_BuffersFramesWithoutDecoding`、`ReadUtf8LineAsync_EnforcesByteLimitBeforeGrowth`、`ReadUtf8LineAsync_HandlesCrLfAcrossBufferBoundary` diff --git a/changelog.d/unreleased/+initial-full-index-symbol-worker-streaming.changed.md b/changelog.d/unreleased/+initial-full-index-symbol-worker-streaming.changed.md new file mode 100644 index 0000000000..4708704cde --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-symbol-worker-streaming.changed.md @@ -0,0 +1,13 @@ +--- +category: changed +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs +--- + +## English + +- **Cold indexing streams symbol-worker requests** — Source content is now JSON-escaped through bounded pooled buffers directly into the worker pipe, avoiding a source-sized request byte array for every language and every parallel extraction worker. + +## 日本語 + +- **初回 index の symbol-worker request を streaming 化** — source content を bounded pooled buffer で JSON escape しながら worker pipe へ直接書き込み、全言語・各並列 extraction worker で source 規模の request byte array を作らないようにしました。 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs index 4082f549ca..26c3dbb605 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs @@ -1,6 +1,8 @@ +using System.Buffers; using System.Diagnostics; using System.Globalization; using System.Text; +using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; using CodeIndex.Cli; @@ -95,7 +97,6 @@ internal SymbolExtractionWorkerResult Invoke( hasOversizeLine, conflictMarkerLine, symlinkPolicy); - var requestUtf8 = JsonSerializer.SerializeToUtf8Bytes(request, SymbolExtractionWorker.JsonOptions); var waitMilliseconds = GetRemainingWaitMilliseconds(stopwatch, callbackBudget); if (waitMilliseconds <= 0) { @@ -110,7 +111,11 @@ internal SymbolExtractionWorkerResult Invoke( process!.StandardOutput.BaseStream, maxProtocolLineBytes, cancellationToken); - sendTask = SendRequestAsync(process.StandardInput.BaseStream, requestUtf8); + sendTask = SendRequestAsync( + process.StandardInput.BaseStream, + request, + maxProtocolLineBytes, + cancellationToken); } catch (Exception ex) { @@ -321,11 +326,105 @@ private static SymbolExtractionWorkerResult TimedOut(long durationMs, string? wo DurationMs: Math.Max(0, durationMs), Symbols: null); - private static async Task SendRequestAsync(Stream input, ReadOnlyMemory requestUtf8) + internal static async Task SendRequestAsync( + Stream input, + SymbolExtractionWorker.WorkerRequest request, + int maxProtocolLineBytes, + CancellationToken cancellationToken = default) { - await input.WriteAsync(requestUtf8).ConfigureAwait(false); - await input.WriteAsync(ProtocolLineTerminator).ConfigureAwait(false); - await input.FlushAsync().ConfigureAwait(false); + const int escapedCharacterBufferSize = 4 * 1024; + var metadata = SymbolExtractionWorker.WorkerRequestMetadata.From(request); + var metadataUtf8 = JsonSerializer.SerializeToUtf8Bytes( + metadata, + SymbolExtractionWorker.JsonOptions); + var writtenBytes = 0; + writtenBytes = await WriteBoundedRequestBytesAsync( + input, + metadataUtf8.AsMemory(0, metadataUtf8.Length - 1), + writtenBytes, + request.Content.Length, + maxProtocolLineBytes, + cancellationToken).ConfigureAwait(false); + writtenBytes = await WriteBoundedRequestBytesAsync( + input, + SymbolExtractionWorker.RequestContentPrefix, + writtenBytes, + request.Content.Length, + maxProtocolLineBytes, + cancellationToken).ConfigureAwait(false); + + var escapedCharacters = ArrayPool.Shared.Rent(escapedCharacterBufferSize); + var escapedUtf8 = ArrayPool.Shared.Rent( + Encoding.UTF8.GetMaxByteCount(escapedCharacterBufferSize)); + try + { + var contentOffset = 0; + while (contentOffset < request.Content.Length) + { + cancellationToken.ThrowIfCancellationRequested(); + var status = JavaScriptEncoder.Default.Encode( + request.Content.AsSpan(contentOffset), + escapedCharacters.AsSpan(0, escapedCharacterBufferSize), + out var charactersConsumed, + out var charactersWritten, + isFinalBlock: true); + if (charactersConsumed == 0 && charactersWritten == 0) + throw new JsonException("Symbol worker request content could not be JSON-escaped."); + + contentOffset += charactersConsumed; + var utf8BytesWritten = Encoding.UTF8.GetBytes( + escapedCharacters.AsSpan(0, charactersWritten), + escapedUtf8); + writtenBytes = await WriteBoundedRequestBytesAsync( + input, + escapedUtf8.AsMemory(0, utf8BytesWritten), + writtenBytes, + request.Content.Length, + maxProtocolLineBytes, + cancellationToken).ConfigureAwait(false); + + if (status == OperationStatus.Done) + break; + if (status != OperationStatus.DestinationTooSmall) + throw new JsonException("Symbol worker request content contains invalid UTF-16."); + } + } + finally + { + ArrayPool.Shared.Return(escapedCharacters, clearArray: true); + ArrayPool.Shared.Return(escapedUtf8, clearArray: true); + } + + _ = await WriteBoundedRequestBytesAsync( + input, + SymbolExtractionWorker.RequestSuffix, + writtenBytes, + request.Content.Length, + maxProtocolLineBytes, + cancellationToken).ConfigureAwait(false); + await input.WriteAsync(ProtocolLineTerminator, cancellationToken).ConfigureAwait(false); + await input.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static async ValueTask WriteBoundedRequestBytesAsync( + Stream output, + ReadOnlyMemory bytes, + int writtenBytes, + int contentCharacters, + int maxProtocolLineBytes, + CancellationToken cancellationToken) + { + if (bytes.Length > maxProtocolLineBytes - writtenBytes) + { + throw new BoundedLineLengthException( + contentCharacters, + writtenBytes, + maxProtocolLineBytes, + maxProtocolLineBytes); + } + + await output.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + return writtenBytes + bytes.Length; } private bool WaitForTask(Task task, int milliseconds, CancellationToken cancellationToken, out Exception? exception) @@ -532,6 +631,8 @@ internal static class SymbolExtractionWorker private const string TestDelayMillisecondsOption = "--test-delay-ms"; private const string TestConsoleStdoutOption = "--test-console-stdout"; private const int CapturedConsoleMaxChars = 32 * 1024; + internal static readonly byte[] RequestContentPrefix = ",\"Content\":\""u8.ToArray(); + internal static readonly byte[] RequestSuffix = "\"}"u8.ToArray(); private static readonly object PatternConfigDiscoveryGate = new(); private static WorkerPatternConfigDiscoveryCache patternConfigDiscoveryCache = new(); @@ -1098,6 +1199,28 @@ internal sealed record WorkerRequest( int? ConflictMarkerLine = null, FileIndexer.SymlinkPolicy SymlinkPolicy = FileIndexer.SymlinkPolicy.All); + internal sealed record WorkerRequestMetadata( + long FileId, + string? Lang, + string FilePath, + string ProjectRoot, + bool ContentIsNormalized, + bool? HasOversizeLine, + int? ConflictMarkerLine, + FileIndexer.SymlinkPolicy SymlinkPolicy) + { + internal static WorkerRequestMetadata From(WorkerRequest request) => + new( + request.FileId, + request.Lang, + request.FilePath, + request.ProjectRoot, + request.ContentIsNormalized, + request.HasOversizeLine, + request.ConflictMarkerLine, + request.SymlinkPolicy); + } + internal sealed record WorkerResponse( List? Symbols, string? WorkerError, @@ -1116,5 +1239,6 @@ private sealed record WorkerOptions( [JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)] [JsonSerializable(typeof(SymbolExtractionWorker.WorkerRequest))] +[JsonSerializable(typeof(SymbolExtractionWorker.WorkerRequestMetadata))] [JsonSerializable(typeof(SymbolExtractionWorker.WorkerResponse))] internal partial class SymbolExtractionWorkerJsonContext : JsonSerializerContext; diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 12fd15f65d..0c8e5e3c30 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -560,12 +560,12 @@ public void SymbolExtractionWorker_Utf8RequestsPreserveUnicodeAcrossLanguages() { var cases = new[] { - (Lang: "csharp", Extension: ".cs", Content: "// 顧客\npublic class Customer { }\n"), - (Lang: "java", Extension: ".java", Content: "// 顧客\npublic class Customer { }\n"), - (Lang: "typescript", Extension: ".ts", Content: "// 顧客\nexport class Customer { }\n"), - (Lang: "python", Extension: ".py", Content: "# 顧客\nclass Customer:\n pass\n"), - (Lang: "go", Extension: ".go", Content: "// 顧客\ntype Customer struct {}\n"), - (Lang: "rust", Extension: ".rs", Content: "// 顧客\npub struct Customer {}\n"), + (Lang: "csharp", Extension: ".cs", Content: "// 顧客 \\\"引用\\\" C:\\\\tmp\t列\r\npublic class Customer { }\n"), + (Lang: "java", Extension: ".java", Content: "// 顧客 \\\"引用\\\" C:\\\\tmp\t列\r\npublic class Customer { }\n"), + (Lang: "typescript", Extension: ".ts", Content: "// 顧客 \\\"引用\\\" C:\\\\tmp\t列\r\nexport class Customer { }\n"), + (Lang: "python", Extension: ".py", Content: "# 顧客 \\\"引用\\\" C:\\\\tmp\t列\r\nclass Customer:\n pass\n"), + (Lang: "go", Extension: ".go", Content: "// 顧客 \\\"引用\\\" C:\\\\tmp\t列\r\ntype Customer struct {}\n"), + (Lang: "rust", Extension: ".rs", Content: "// 顧客 \\\"引用\\\" C:\\\\tmp\t列\r\npub struct Customer {}\n"), }; using var worker = new SymbolExtractionWorkerClient(); var warmup = worker.Invoke( @@ -602,6 +602,102 @@ public void SymbolExtractionWorker_Utf8RequestsPreserveUnicodeAcrossLanguages() } } + [Fact] + public async Task SymbolExtractionWorker_RequestStreamingPreservesJsonAndBoundsAllocation() + { + var request = new SymbolExtractionWorker.WorkerRequest( + 17, + "csharp", + "first\r\n\\\"quoted\\\"\\path\t顧客\nlast", + "/workspace/顧客.cs", + "/workspace", + ContentIsNormalized: true, + HasOversizeLine: false, + ConflictMarkerLine: 9, + FileIndexer.SymlinkPolicy.Internal); + using var frame = new MemoryStream(); + + await SymbolExtractionWorkerClient.SendRequestAsync( + frame, + request, + maxProtocolLineBytes: 64 * 1024); + + var frameBytes = frame.ToArray(); + Assert.Equal((byte)'{', frameBytes[0]); + Assert.Equal((byte)'\n', frameBytes[^1]); + Assert.Equal(1, frameBytes.Count(value => value == (byte)'\n')); + var roundTripped = JsonSerializer.Deserialize( + frameBytes.AsSpan(0, frameBytes.Length - 1), + SymbolExtractionWorker.JsonOptions); + Assert.Equal(request, roundTripped); + + var largeRequest = request with { Content = new string('"', 512 * 1024) }; + using (var warmup = new CountingWriteStream()) + { + await SymbolExtractionWorkerClient.SendRequestAsync( + warmup, + largeRequest, + maxProtocolLineBytes: 8 * 1024 * 1024); + } + + using var counting = new CountingWriteStream(); + var allocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + await SymbolExtractionWorkerClient.SendRequestAsync( + counting, + largeRequest, + maxProtocolLineBytes: 8 * 1024 * 1024); + var allocatedBytes = GC.GetAllocatedBytesForCurrentThread() - allocatedBefore; + + Assert.True( + allocatedBytes < 128 * 1024, + $"Expected bounded request streaming allocation, measured {allocatedBytes:N0} bytes."); + Assert.True(counting.MaxWriteSize <= 16 * 1024); + + using var capped = new CountingWriteStream(); + await Assert.ThrowsAsync(() => + SymbolExtractionWorkerClient.SendRequestAsync( + capped, + request with { Content = new string('"', 256) }, + maxProtocolLineBytes: 128)); + Assert.True(capped.BytesWritten <= 128); + } + + private sealed class CountingWriteStream : Stream + { + internal long BytesWritten { get; private set; } + internal int MaxWriteSize { get; private set; } + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => BytesWritten; + public override long Position + { + get => BytesWritten; + set => throw new NotSupportedException(); + } + + public override void Flush() { } + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => Record(count); + public override ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + Record(buffer.Length); + return ValueTask.CompletedTask; + } + + private void Record(int count) + { + BytesWritten += count; + MaxWriteSize = Math.Max(MaxWriteSize, count); + } + } + [Fact] public void SymbolExtractionWorker_NimIdentityKeySurvivesProtocolRoundTrip_Issue4738() { From f52e5f31bfe133dc8c13cba9a54b62fe9cb2fa4c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 29 Aug 2026 19:12:45 +0900 Subject: [PATCH 05/11] Materialize shared reference scope candidates --- DEVELOPER_GUIDE.md | 12 + TESTING_GUIDE.md | 4 +- ...ull-index-reference-scope-facts.changed.md | 13 ++ .../DbWriter.ReferenceGraphRefreshScope.cs | 2 +- src/CodeIndex/Database/DbWriter.References.cs | 211 +++++++----------- tests/CodeIndex.Tests/DatabaseTests.cs | 154 ++++++++++++- 6 files changed, 261 insertions(+), 135 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-reference-scope-facts.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 3c35842a2c..85b719cc0e 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -478,6 +478,12 @@ retained rebuilds use the complete C# symbol-fact population. Property-receiver normalization must likewise drive from flagged reference facts and the target fact primary key; scoped target materialization is restricted to its lookup-name set. +Language-independent scope ranks 1–4 must build their shared reference/name/language +candidate relation once in a materialized CTE, assign each reference/symbol pair its best +applicable rank, and retain every candidate tied at the reference's minimum rank. Keep source +symbol attribution optional so the rank-3 same-file fallback survives missing source identity, +and keep scoped refreshes driven from dirty reference IDs into the reference primary key. + After rank 0–4 candidate construction, graph finalization materializes the distinct matching reference IDs into a compact `WITHOUT ROWID` TEMP table. All language-independent and C# rank-5 fallbacks consult that set instead of probing @@ -4700,6 +4706,12 @@ lookup-name 集合だけに限定し、identity fact もその限定済み集合 C# symbol fact の全対象を使います。property-receiver normalization も flag 済み reference fact と target fact の primary key から駆動し、scoped target materialization は lookup-name 集合だけに限定してください。 +言語共通の scope rank 1〜4 は、共有する reference / name / language candidate relation を +materialized CTE で1回だけ構築し、reference / symbol pair ごとの最良rankを割り当てたうえで、 +reference ごとの最小rankに同順位の全candidateを保持します。source identity が不明でも rank 3 の +same-file fallback を残し、scoped refresh は dirty reference ID から reference primary key へ +駆動する契約を維持してください。 + rank 0〜4 の candidate 構築後は、一致した reference ID の distinct 集合を compact な `WITHOUT ROWID` TEMP table に materialize します。言語共通および C# の rank 5 fallback は 巨大な物理 candidate table ではなくこの集合を参照し、永続化される symbol ごとの candidate 行と diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index b503ce0178..955807941e 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -817,7 +817,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` fixes atomic reference-line windows to a worst-case arithmetic bound. Keep it paired with whole-statement grouping, 32-statement caps, materialization-window-boundary context reuse, rollback, and cancellation tests so the materializer remains the only tuple-hash pass without changing persistence boundaries. `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. - `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` also fixes the rank-5 lower-rank guard at one compact TEMP row per matched reference: scoped materialization must drive from dirty IDs into the candidate primary key, and each of the five rank-5 fallbacks must use a TEMP primary-key seek. The C# type fallback must materialize physical members, unique logical families, and matched families in that order before its final physical expansion; retain primary-key seeks for scoped symbols, references, and facts. `CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember` proves that same-identity partial declarations still produce every physical candidate while a conflicting identity suppresses the whole rank-5 family across scoped, full, and retained refreshes. Keep the remaining full/scoped/retained resolution oracles beside these structural guards so physical candidate rows and multi-language ambiguity remain unchanged. + `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` also fixes language-independent ranks 1–4 at one materialized candidate relation plus one minimum-rank relation, while scoped SQL must drive that relation from dirty IDs into reference primary keys. It keeps the rank-5 lower-rank guard at one compact TEMP row per matched reference, and each of the five rank-5 fallbacks must use a TEMP primary-key seek. `ReferenceScopeCandidates_MaterializeMinimumRankAndPreserveTies` proves C# and Python retain rank 1–4 precedence, every minimum-rank tie, the source-less same-file fallback, and identical scoped/full/retained results. The C# type fallback must materialize physical members, unique logical families, and matched families in that order before its final physical expansion; retain primary-key seeks for scoped symbols, references, and facts. `CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember` proves that same-identity partial declarations still produce every physical candidate while a conflicting identity suppresses the whole rank-5 family across scoped, full, and retained refreshes. Keep the remaining full/scoped/retained resolution oracles beside these structural guards so physical candidate rows and multi-language ambiguity remain unchanged. `DatabaseTests.CSharpPropertyReceiverNormalization_SeeksFactBackedReferencesAndTargets` requires both normalization updates to seek flagged reference IDs and the primary-keyed field/property target facts rather than scan all references or persistent target symbols. Keep its full/scoped/retained stage-order assertions and property-resolution fixtures paired so lookup-name scoping cannot change inherited-member semantics. `FreshReferenceResolutionTests.ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope` fixes target-family key construction to one per target symbol and candidate resolution to TEMP primary-key facts across fresh, full, differential, scoped, and retained paths. Its legacy-null-key and C#/Python oracle coverage preserves resolved IDs, exact keys, grouped families, ambiguity, and self-reference semantics. - `HotspotReferenceAggregateTests.cs` @@ -1968,7 +1968,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` はatomic reference-line windowを最悪ケースの算術境界へ固定します。materializerだけをtuple-hash passとして保ちつつ永続化境界を変えないよう、whole-statement grouping、32-statement cap、materialization window境界をまたぐcontext再利用、rollback、cancellation testと対で維持してください。 `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 - `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` は、rank 5 の下位rank guardも、一致したreferenceごとに1行のcompact TEMP集合へ固定します。scoped materializationはdirty IDからcandidate primary keyをseekし、5つのrank 5 fallbackはそれぞれTEMP primary-key seekを使う必要があります。C# type fallbackは最終の物理展開より前に、物理member、一意な論理family、一致familyの順でmaterializeし、scoped symbol / reference / factのprimary-key seekを維持してください。`CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember`は、同一identityのpartial宣言が引き続き全物理candidateを生成し、競合identityがscoped / full / retained refreshを横断してrank 5 family全体を抑止することを証明します。物理candidate行と多言語ambiguityが変わらないよう、残りのfull / scoped / retained resolution oracleもこれらの構造guardと対で維持してください。 + `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` は、言語共通のrank 1〜4を1つのmaterialized candidate relationと1つのminimum-rank relationへ固定し、scoped SQLがdirty IDからreference primary keyへ駆動することも要求します。rank 5の下位rank guardは、一致したreferenceごとに1行のcompact TEMP集合を維持し、5つのrank 5 fallbackはそれぞれTEMP primary-key seekを使う必要があります。`ReferenceScopeCandidates_MaterializeMinimumRankAndPreserveTies`はC# / Pythonでrank 1〜4の優先順位、最小rankの全同順位、source不明時のsame-file fallback、scoped / full / retainedの同一結果を証明します。C# type fallbackは最終の物理展開より前に、物理member、一意な論理family、一致familyの順でmaterializeし、scoped symbol / reference / factのprimary-key seekを維持してください。`CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember`は、同一identityのpartial宣言が引き続き全物理candidateを生成し、競合identityがscoped / full / retained refreshを横断してrank 5 family全体を抑止することを証明します。物理candidate行と多言語ambiguityが変わらないよう、残りのfull / scoped / retained resolution oracleもこれらの構造guardと対で維持してください。 `DatabaseTests.CSharpPropertyReceiverNormalization_SeeksFactBackedReferencesAndTargets` は、2つのnormalization updateが全referenceや永続target symbolをscanせず、flag済みreference IDとprimary-keyed field / property target factをseekすることを要求します。lookup-name scopeが継承member semanticsを変えないよう、full / scoped / retainedのstage-order assertionとproperty-resolution fixtureを対で維持してください。 `FreshReferenceResolutionTests.ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope` は、target-family key構築をtarget symbolごと1回に限定し、fresh / full / differential / scoped / retainedのcandidate resolutionがTEMP primary-key factを使う契約を固定します。legacy null-keyとC# / Python oracle coverageにより、resolved ID、exact key、group family、ambiguity、self-reference semanticsを維持します。 - `HotspotReferenceAggregateTests.cs` diff --git a/changelog.d/unreleased/+initial-full-index-reference-scope-facts.changed.md b/changelog.d/unreleased/+initial-full-index-reference-scope-facts.changed.md new file mode 100644 index 0000000000..a17e5db583 --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-reference-scope-facts.changed.md @@ -0,0 +1,13 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.References.cs +--- + +## English + +- **Cold graph finalization reuses scope candidates** — Language-independent reference ranks 1–4 now materialize their shared candidate relation once and select each reference's minimum rank from it, avoiding three repeated reference/name/language scans while retaining all best-rank ties. + +## 日本語 + +- **初回 graph 確定で scope candidate を再利用** — 言語共通の reference rank 1〜4 は共有 candidate relation を1回だけ materialize し、そこから reference ごとの最小rankを選ぶようになりました。最良rankの全同順位を維持しながら、reference / name / language の重複走査を3回削減します。 diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index b0cad0ace6..d494b91aa5 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -292,7 +292,7 @@ private static string BuildScopedReferenceCandidatesSql() const string fullCSharpTypeNamePredicateSql = "AND type_symbol.name_folded IS NOT NULL"; const string fullLowerRankCandidateSourceSql = "FROM symbol_reference_candidates AS lower_rank_candidate"; - const int expectedReferenceSourceCount = 15; + const int expectedReferenceSourceCount = 12; if (CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullDeleteSql) != 1 || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullReferenceSourceSql) diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index a3037aeda6..9b642b8959 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -1335,135 +1335,88 @@ SELECT 1 FROM symbol_reference_candidates AS existing ); INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) - SELECT r.id, s.id, 1 - FROM symbol_references AS r - JOIN files AS source_file ON source_file.id = r.file_id - JOIN symbols AS s - ON s.name_folded IN ( - r.symbol_name_folded, - CASE WHEN source_file.lang = 'csharp' AND r.reference_kind = 'attribute' - THEN r.symbol_name_folded || 'attribute' END - ) - JOIN files AS target_file ON target_file.id = s.file_id - JOIN symbols AS source ON source.id = r.source_symbol_id - WHERE ( - (source_file.lang = target_file.lang - AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) - OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) - ) - AND {CSharpTypeReferenceCandidatePredicateSql} - AND source_file.lang <> 'markdown' - AND ( - r.target_qualifier IS NULL - OR (source_file.lang = 'csharp' - AND r.target_qualifier = char(31) || 'csharp_nonlocal') - ) - AND s.file_id = r.file_id - AND source.container_name IS NOT NULL - AND source.container_name <> '' - AND ( - s.container_name = source.container_name COLLATE NOCASE - OR s.container_qualified_name = source.container_qualified_name COLLATE NOCASE - ) - AND NOT EXISTS ( - SELECT 1 FROM symbol_reference_candidates AS existing - WHERE existing.reference_id = r.id - ); - - INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) - SELECT r.id, s.id, 2 - FROM symbol_references AS r - JOIN files AS source_file ON source_file.id = r.file_id - JOIN symbols AS s - ON s.name_folded IN ( - r.symbol_name_folded, - CASE WHEN source_file.lang = 'csharp' AND r.reference_kind = 'attribute' - THEN r.symbol_name_folded || 'attribute' END - ) - JOIN files AS target_file ON target_file.id = s.file_id - JOIN symbols AS source ON source.id = r.source_symbol_id - WHERE ( - (source_file.lang = target_file.lang - AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) - OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) - ) - AND {CSharpTypeReferenceCandidatePredicateSql} - AND source_file.lang <> 'markdown' - AND ( - r.target_qualifier IS NULL - OR (source_file.lang = 'csharp' - AND r.target_qualifier = char(31) || 'csharp_nonlocal') - ) - AND (source_file.lang <> 'dependency_lock' OR s.file_id = r.file_id) - AND source.container_qualified_name IS NOT NULL - AND source.container_qualified_name <> '' - AND s.container_qualified_name = source.container_qualified_name COLLATE NOCASE - AND NOT EXISTS ( - SELECT 1 FROM symbol_reference_candidates AS existing - WHERE existing.reference_id = r.id - ); - - INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) - SELECT r.id, s.id, 3 - FROM symbol_references AS r - JOIN files AS source_file ON source_file.id = r.file_id - JOIN symbols AS s - ON s.name_folded IN ( - r.symbol_name_folded, - CASE WHEN source_file.lang = 'csharp' AND r.reference_kind = 'attribute' - THEN r.symbol_name_folded || 'attribute' END - ) - JOIN files AS target_file ON target_file.id = s.file_id - WHERE ( - (source_file.lang = target_file.lang - AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) - OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) - ) - AND {CSharpTypeReferenceCandidatePredicateSql} - AND source_file.lang <> 'markdown' - AND ( - r.target_qualifier IS NULL - OR (source_file.lang = 'csharp' - AND r.target_qualifier = char(31) || 'csharp_nonlocal') - ) - AND s.file_id = r.file_id - AND NOT EXISTS ( - SELECT 1 FROM symbol_reference_candidates AS existing - WHERE existing.reference_id = r.id - ); - - INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) - SELECT r.id, s.id, 4 - FROM symbol_references AS r - JOIN files AS source_file ON source_file.id = r.file_id - JOIN symbols AS s - ON s.name_folded IN ( - r.symbol_name_folded, - CASE WHEN source_file.lang = 'csharp' AND r.reference_kind = 'attribute' - THEN r.symbol_name_folded || 'attribute' END - ) - JOIN files AS target_file ON target_file.id = s.file_id - JOIN symbols AS source ON source.id = r.source_symbol_id - WHERE ( - (source_file.lang = target_file.lang - AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) - OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) - ) - AND {CSharpTypeReferenceCandidatePredicateSql} - AND source_file.lang <> 'markdown' - AND ( - r.target_qualifier IS NULL - OR (source_file.lang = 'csharp' - AND r.target_qualifier = char(31) || 'csharp_nonlocal') - ) - AND (source_file.lang <> 'dependency_lock' OR s.file_id = r.file_id) - AND source.container_name IS NOT NULL - AND source.container_name <> '' - AND s.container_name = source.container_name COLLATE NOCASE - AND NOT EXISTS ( - SELECT 1 FROM symbol_reference_candidates AS existing - WHERE existing.reference_id = r.id - ); + -- Ranks 1-4 share the same reference/name/language candidate relation. Materialize + -- that relation once, assign each pair its best applicable rank, then retain every + -- candidate tied at the reference's minimum rank. A LEFT JOIN preserves the rank-3 + -- same-file fallback when source-symbol attribution is unavailable. + -- rank 1-4で共通するreference/name/language候補を一度だけmaterializeし、各pairの + -- 最良rankを求めた後、referenceごとの最小rankに同順位の全candidateを保持する。 + -- source symbol不明でもrank 3のsame-file fallbackを残すためLEFT JOINを使う。 + WITH scope_candidates(reference_id, symbol_id, scope_rank) AS MATERIALIZED ( + SELECT r.id, + s.id, + CASE + WHEN s.file_id = r.file_id + AND source.container_name IS NOT NULL + AND source.container_name <> '' + AND ( + s.container_name = source.container_name COLLATE NOCASE + OR s.container_qualified_name = + source.container_qualified_name COLLATE NOCASE + ) THEN 1 + WHEN source.container_qualified_name IS NOT NULL + AND source.container_qualified_name <> '' + AND s.container_qualified_name = + source.container_qualified_name COLLATE NOCASE THEN 2 + WHEN s.file_id = r.file_id THEN 3 + WHEN source.container_name IS NOT NULL + AND source.container_name <> '' + AND s.container_name = source.container_name COLLATE NOCASE THEN 4 + END + FROM symbol_references AS r + JOIN files AS source_file ON source_file.id = r.file_id + JOIN symbols AS s + ON s.name_folded IN ( + r.symbol_name_folded, + CASE WHEN source_file.lang = 'csharp' AND r.reference_kind = 'attribute' + THEN r.symbol_name_folded || 'attribute' END + ) + JOIN files AS target_file ON target_file.id = s.file_id + LEFT JOIN symbols AS source ON source.id = r.source_symbol_id + WHERE ( + (source_file.lang = target_file.lang + AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) + OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) + ) + AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' + AND ( + r.target_qualifier IS NULL + OR (source_file.lang = 'csharp' + AND r.target_qualifier = char(31) || 'csharp_nonlocal') + ) + AND (source_file.lang <> 'dependency_lock' OR s.file_id = r.file_id) + AND ( + s.file_id = r.file_id + OR ( + source.container_qualified_name IS NOT NULL + AND source.container_qualified_name <> '' + AND s.container_qualified_name = + source.container_qualified_name COLLATE NOCASE + ) + OR ( + source.container_name IS NOT NULL + AND source.container_name <> '' + AND s.container_name = source.container_name COLLATE NOCASE + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM symbol_reference_candidates AS existing + WHERE existing.reference_id = r.id + ) + ), + minimum_scopes(reference_id, scope_rank) AS MATERIALIZED ( + SELECT reference_id, MIN(scope_rank) + FROM scope_candidates + GROUP BY reference_id + ) + SELECT candidate.reference_id, + candidate.symbol_id, + candidate.scope_rank + FROM scope_candidates AS candidate + JOIN minimum_scopes AS minimum_scope + ON minimum_scope.reference_id = candidate.reference_id + AND minimum_scope.scope_rank = candidate.scope_rank; -- Rank-5 fallbacks only need to know whether a lower rank matched. Keep that -- one-row-per-reference fact compact instead of probing the much larger diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index ee5733a540..dfaf1940e1 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -361,6 +361,17 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() Assert.Equal(["full", "scoped", "retained"], candidateStages.Select(static stage => stage.Scope)); foreach (var (scope, sql) in candidateStages) { + Assert.Equal(1, CountOccurrences(sql, "scope_candidates(")); + Assert.Equal(1, CountOccurrences(sql, "minimum_scopes(")); + Assert.Contains( + "scope_candidates(reference_id, symbol_id, scope_rank) AS MATERIALIZED", + sql, + StringComparison.Ordinal); + Assert.Contains( + "minimum_scopes(reference_id, scope_rank) AS MATERIALIZED", + sql, + StringComparison.Ordinal); + Assert.Contains("LEFT JOIN symbols AS source", sql, StringComparison.Ordinal); Assert.Contains("temp.csharp_type_identity_facts", sql, StringComparison.Ordinal); Assert.Contains("temp.csharp_constructor_identity_facts", sql, StringComparison.Ordinal); Assert.Contains( @@ -471,6 +482,143 @@ static int CountOccurrences(string text, string value) } } + [Theory] + [InlineData("csharp", ".cs")] + [InlineData("python", ".py")] + public void ReferenceScopeCandidates_MaterializeMinimumRankAndPreserveTies( + string language, + string extension) + { + var callerFileId = UpsertTestFileWithLanguage( + $"scope/{language}/Caller{extension}", + language, + $"{language}-scope-caller"); + var targetFileId = UpsertTestFileWithLanguage( + $"scope/{language}/Targets{extension}", + language, + $"{language}-scope-targets"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = callerFileId, + Kind = "function", + Name = "Caller", + Line = 1, + StartLine = 1, + EndLine = 20, + ContainerName = "Outer", + ContainerQualifiedName = "Demo.Outer", + }, + Target(callerFileId, "TieTarget", 30, "Outer", "Demo.Outer"), + Target(callerFileId, "TieTarget", 31, "Other", "Demo.Outer"), + Target(targetFileId, "TieTarget", 1, "Other", "Demo.Outer"), + Target(targetFileId, "QualifiedTarget", 2, "Other", "Demo.Outer"), + Target(callerFileId, "QualifiedTarget", 32, "Other", "Other.Qualified"), + Target(callerFileId, "FileTarget", 33, "Other", "Other.File"), + Target(targetFileId, "FileTarget", 3, "Outer", "Other.Outer"), + Target(targetFileId, "ContainerTarget", 4, "Outer", "Other.Outer"), + Target(targetFileId, "ContainerTarget", 5, "Other", "Other.Container"), + Target(callerFileId, "NoSourceTarget", 34, "Other", "Other.NoSource"), + ]); + _writer.InsertReferences([ + Reference("TieTarget", line: 5, containerName: "Caller"), + Reference("QualifiedTarget", line: 6, containerName: "Caller"), + Reference("FileTarget", line: 7, containerName: "Caller"), + Reference("ContainerTarget", line: 8, containerName: "Caller"), + Reference("NoSourceTarget", line: 25, containerName: null), + ], refreshMutualRecursionFlags: false); + + _writer.RefreshMutualRecursionFlags(); + const string expected = "5:1:30|5:1:31|6:2:2|7:3:33|8:4:4|25:3:34"; + Assert.Equal(expected, ReadCandidates()); + + using (var scope = _writer.BeginReferenceGraphRefreshScope()) + { + using var transaction = _writer.BeginTransaction(); + var distractorFileId = _writer.InsertNewFile(new FileRecord + { + Path = $"scope/{language}/Distractors{extension}", + Lang = language, + Size = 100, + Lines = 10, + Modified = new DateTime(2026, 8, 29, 0, 0, 0, DateTimeKind.Utc), + Checksum = $"{language}-scope-distractors", + }); + _writer.InsertSymbols( + [ + Target(distractorFileId, "TieTarget", 1, "Elsewhere", "Elsewhere.Type"), + Target(distractorFileId, "QualifiedTarget", 2, "Elsewhere", "Elsewhere.Type"), + Target(distractorFileId, "FileTarget", 3, "Elsewhere", "Elsewhere.Type"), + Target(distractorFileId, "ContainerTarget", 4, "Elsewhere", "Elsewhere.Type"), + Target(distractorFileId, "NoSourceTarget", 5, "Elsewhere", "Elsewhere.Type"), + ]); + transaction.Commit(); + _writer.RefreshMutualRecursionFlags(); + } + + Assert.Equal(expected, ReadCandidates()); + var scopedSnapshot = ReadReferenceGraphSemanticSnapshot(); + + _writer.RefreshMutualRecursionFlags(); + Assert.Equal(expected, ReadCandidates()); + var fullSnapshot = ReadReferenceGraphSemanticSnapshot(); + Assert.Equal(scopedSnapshot, fullSnapshot); + + using (var transaction = _db.Connection.BeginTransaction()) + { + DbWriter.RebuildRetainedReferenceGraph( + _db.Connection, + transaction, + CancellationToken.None); + transaction.Commit(); + } + + Assert.Equal(expected, ReadCandidates()); + Assert.Equal(fullSnapshot, ReadReferenceGraphSemanticSnapshot()); + + SymbolRecord Target( + long fileId, + string name, + int line, + string containerName, + string containerQualifiedName) + => new() + { + FileId = fileId, + Kind = "property", + Name = name, + Line = line, + StartLine = line, + EndLine = line, + ContainerName = containerName, + ContainerQualifiedName = containerQualifiedName, + }; + + ReferenceRecord Reference(string name, int line, string? containerName) + => new() + { + FileId = callerFileId, + SymbolName = name, + ReferenceKind = "reference", + Line = line, + Column = 1, + Context = name, + ContainerName = containerName, + }; + + string ReadCandidates() + => ExecuteScalarString(""" + SELECT COALESCE(group_concat(entry, '|'), '') + FROM ( + SELECT reference.line || ':' || candidate.scope_rank || ':' || target.line AS entry + FROM symbol_reference_candidates AS candidate + JOIN symbol_references AS reference ON reference.id = candidate.reference_id + JOIN symbols AS target ON target.id = candidate.symbol_id + ORDER BY reference.line, candidate.scope_rank, target.line + ) + """); + } + [Fact] public void CSharpInstantiationFallback_SetBasedFamiliesPreserveSemanticBoundaries() { @@ -1668,7 +1816,7 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() StringComparison.Ordinal)); Assert.True(fullMaterializationIndex >= 0); Assert.Equal( - 9, + 6, fullCandidateStatements[..fullMaterializationIndex].Count(static statement => statement.StartsWith( "INSERT INTO symbol_reference_candidates", @@ -1695,7 +1843,7 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() candidateSql, StringComparison.Ordinal); Assert.Equal( - 15, + 12, candidateSql.Split( "FROM temp.reference_graph_dirty_references AS dirty_reference", StringSplitOptions.None).Length - 1); @@ -1732,7 +1880,7 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() "INSERT INTO symbol_reference_candidates", StringComparison.Ordinal)) .ToArray(); - Assert.Equal(14, candidateInserts.Length); + Assert.Equal(11, candidateInserts.Length); var candidatePlans = new List(); foreach (var statement in candidateInserts) { From 88ccb7723696595d20e2e3ba73ba40c5e1442a3c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 29 Aug 2026 21:10:14 +0900 Subject: [PATCH 06/11] Assign fresh reference line IDs without returning scans --- DEVELOPER_GUIDE.md | 4 +- TESTING_GUIDE.md | 10 +- ...-index-fresh-reference-line-ids.changed.md | 14 + .../DbWriter.AuthoritativeFreshBulkInsert.cs | 226 +++++------- ...riter.AuthoritativeFreshIdentityInsert.cs} | 106 +++--- src/CodeIndex/Database/DbWriter.BatchSql.cs | 8 +- .../Database/DbWriter.ReferenceSql.cs | 30 ++ src/CodeIndex/Database/DbWriter.References.cs | 4 +- src/CodeIndex/Database/DbWriter.cs | 2 + .../AuthoritativeFreshRawBulkInsertTests.cs | 346 ++++++++++++------ tests/CodeIndex.Tests/DatabaseTests.cs | 86 ++++- .../IndexCommandRunnerFullScanTests.cs | 16 +- ...dRunnerInitialFullIndexPerformanceTests.cs | 2 +- 13 files changed, 544 insertions(+), 310 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-fresh-reference-line-ids.changed.md rename src/CodeIndex/Database/{DbWriter.AuthoritativeFreshReturningInsert.cs => DbWriter.AuthoritativeFreshIdentityInsert.cs} (58%) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 85b719cc0e..33187abb6f 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1470,7 +1470,7 @@ Current stable codes and triggers: | `synchronous=NORMAL` | Under WAL, `NORMAL` avoids per-commit fsync pressure during 500-row indexing batches while preserving database consistency after crashes. | | Caller-owned write batching | Full-scan and other atomic file writes already run inside one caller-owned transaction, so their language-neutral chunk, symbol, issue, reference-line, and reference inserts cap each statement at 32 parameters. Every batch uses compact, one-origin SQLite numeric slots (`?1` through `?N`) in row/column order; this reduces parameter-name resolution work while preserving the existing statement-size, cancellation, and checkpoint contracts. For operations above 500 rows, persistent `db_writer_batch_checkpoint` records are emitted only when progress crosses a 500-row boundary and at completion, avoiding a synchronous log flush for every tiny statement. Public writer APIs retain the SQLite-variable-limit batch shape and their existing per-batch transaction/SAVEPOINT contract. | | Prepared savepoint controls | `DbWriter` leases only a fixed, bounded set of control statements from the connection's prepared-command cache: the first nested `sp_1` SAVEPOINT / RELEASE / ROLLBACK trio used by per-file full-index scopes, the atomic metadata savepoints, and the FTS bulk-load owner savepoint. Every lease rebinds the current outer `SqliteTransaction`; a cacheless writer still creates and disposes one command per call. Depth-two and deeper savepoint names remain dynamic and bypass the cache, and cancellation, rollback, terminal-state, and transaction-gate contracts are unchanged. | -| Authoritative-fresh raw insert scope | After the empty-database CLI path revalidates its authoritative-fresh claim inside the caller-owned transaction, only the extraction pipeline's new-file and fresh reference-line `RETURNING` INSERTs plus its DONE-only chunk, symbol, new-file issue, and atomic fresh-reference INSERTs may bind and execute through SQLitePCLRaw on the provider-owned connection handle. These native positional bindings use a separate 512-parameter ceiling while provider-backed caller-owned writes retain their 32-parameter limit. The scope preserves exact tail/result shapes, batch hooks, row-skip replay, and outer transaction atomicity; a 32-entry LRU retains recurring full and tail statement shapes. The synchronous full-scan persistence consumer and `DbWriter` transaction-owner check keep the non-thread-safe cache single-owner. Every lease resets and clears bindings, errors retain the original step result, cancellation maps SQLite interrupt to `OperationCanceledException`, and all cached statements are finalized before graph/index/FTS work. During this same transaction the three `files_resource_generation_*` triggers are suspended, then recreated only after native statements finalize; the resource-list generation advances exactly once when at least one file was persisted and stays unchanged for an empty repository. Rollback restores both schema and generation atomically. A `RETURNING` lease buffers and validates every positive ID and input ordinal through terminal `DONE` before publishing; a bounded ID set keeps duplicate validation linear as batches grow. Malformed, incomplete, duplicate, failed, or cancelled streams discard the prepared statement, while the caller's per-file savepoint owns data rollback. Replacement, incremental, rebuild, symbols-only, fresh-claim race fallback, MCP, and public writer paths remain on Microsoft.Data.Sqlite with per-mutation generation invalidation. | +| Authoritative-fresh raw insert scope | After the empty-database CLI path revalidates its authoritative-fresh claim inside the caller-owned transaction, only the extraction pipeline's new-file, chunk, symbol, new-file issue, fresh reference-line, and atomic fresh-reference INSERTs may bind and execute through SQLitePCLRaw on the provider-owned connection handle. These native positional bindings use a separate 512-parameter ceiling while provider-backed caller-owned writes retain their 32-parameter limit. Fresh file and reference-line writes are DONE-only: file insertion captures the same connection's positive `last_insert_rowid`, while every reference-line batch reads the greater of `MAX(id)` and `sqlite_sequence.seq`, checks the complete Int64 range, and inserts explicit contiguous IDs with `?1 + input_ordinal`. Reading the floor for every batch preserves AUTOINCREMENT history and observes inserts between batches without retaining rollback-sensitive allocator state. Invalid floors and identity-range overflow fail before the INSERT executes. Every executed fresh identity write validates `sqlite3_changes()` before publishing IDs; a row-count mismatch, constraint, cleanup failure, or cancellation discards the affected prepared statement while the caller's per-file savepoint owns data rollback. The scope preserves exact tail/write-count shapes, batch hooks, row-skip replay, and outer transaction atomicity; a 32-entry LRU retains recurring full and tail statement shapes. The synchronous full-scan persistence consumer and `DbWriter` transaction-owner check keep the non-thread-safe cache single-owner. Every lease resets and clears bindings, errors retain the original step result, cancellation maps SQLite interrupt to `OperationCanceledException`, and all cached statements are finalized before graph/index/FTS work. During this same transaction the three `files_resource_generation_*` triggers are suspended, then recreated only after native statements finalize; the resource-list generation advances exactly once when at least one file was persisted and stays unchanged for an empty repository. Rollback restores both schema and generation atomically. Replacement, incremental, rebuild, symbols-only, fresh-claim race fallback, MCP, and public writer paths remain on Microsoft.Data.Sqlite, retain their established `RETURNING` behavior, and keep per-mutation generation invalidation. | | Authoritative-fresh core secondary indexes | The same revalidated empty-database CLI transaction drops 22 language-neutral secondary indexes on `files`, `chunks`, `file_issues`, and `symbols` before persistence, then builds each B-tree once after every native INSERT statement has finalized and before graph or readiness queries begin. UNIQUE autoindexes remain active, so path and table constraints keep their normal enforcement. `idx_symbols_file` also remains active because fresh-reference insertion resolves each containing source symbol through a correlated per-file lookup; dropping it would turn that hot path into repeated full symbol-table scans. Cancellation or failure leaves restoration to the outer rollback, which atomically restores the pre-load schema; rebuild, incremental, fresh-claim race fallback, and MCP paths retain the indexes throughout their writes. Canonical DDL is shared by schema initialization, opportunistic read migration, and the bulk-load guard so the deferred set cannot drift from the completed database contract. | | Checkpointing | `DbWriter` runs `PRAGMA wal_checkpoint(PASSIVE)` after each outer transaction commit, and SQLite may also checkpoint automatically after the configured 1000-page threshold. Both checkpoint paths are opportunistic: active readers are not blocked, and an uncheckpointed WAL is expected state rather than corruption. | | Checkpoint result contract | Explicit `PRAGMA wal_checkpoint(TRUNCATE)` paths execute a reader and return a structured result containing SQLite's `(busy, log, checkpointed)` values. Non-zero `busy` or positive remaining pages is unsuccessful with a bounded machine reason. `(0, -1, -1)` is SQLite's successful non-WAL no-op. Instance checkpointing, the static read-only-fallback preflight, query diagnostics, top-level status, and nested connection-policy status preserve the same result and counts. Raw exception text and paths must not enter diagnostics. | @@ -5616,7 +5616,7 @@ apply 時は `PRAGMA optimize` を実行します。 | `synchronous=NORMAL` | WAL では `NORMAL` により 500 row 単位の indexing batch ごとの fsync 負荷を避けつつ、crash 後の database consistency を保ちます。 | | caller-owned write batch | full-scan などの atomic file write は既に1つの caller-owned transaction 内で実行されるため、言語共通の chunk、symbol、issue、reference-line、reference insert は statement を32 parameter以下に制限します。すべてのbatchはrow / column順にcompactな1-origin SQLite numeric slot（`?1`〜`?N`）を使い、既存のstatement-size、cancellation、checkpoint契約を保ったままparameter name解決の処理を抑えます。500 rowを超えるoperationでは、永続 `db_writer_batch_checkpoint` を500 row境界をまたいだ時点と完了時だけ出力することで、小さなstatementごとの同期log flushを避けます。public writer API は SQLite variable limit までの batch 形状と既存の batch ごとの transaction / SAVEPOINT 契約を維持します。 | | prepared savepoint control | `DbWriter` は connection の prepared-command cache から固定・有界な control statement だけを借ります。対象は file 単位 full-index scope が使う最初の nested `sp_1` の SAVEPOINT / RELEASE / ROLLBACK、atomic metadata savepoint、FTS bulk-load owner savepoint です。各 lease は現在の outer `SqliteTransaction` へ再 bind し、cache なし writer は従来どおり呼び出しごとに command を作成・破棄します。depth 2 以深の savepoint 名は動的なまま cache を迂回し、cancellation、rollback、terminal state、transaction gate の契約は変更しません。 | -| authoritative-fresh raw insert scope | empty-database CLI経路がcaller-owned transaction内でauthoritative-fresh claimを再検証した後に限り、extraction pipelineのnew-file / fresh reference-line `RETURNING` INSERTと、DONE-onlyなchunk、symbol、new-file issue、atomic fresh-reference INSERTをprovider所有connection handle上のSQLitePCLRawでbind / executeします。scopeは既存の32 parameter statement境界、正確なtail / result形状、batch hook、row-skip replay、outer transaction atomicityを維持し、32-entry LRUで本番25形状すべてを保持します。同期的なfull-scan persistence consumerと`DbWriter`のtransaction owner検査により、非thread-safe cacheはsingle-ownerのままです。各leaseはresetとbinding clearを行い、error時は元のstep結果を保持し、SQLite interruptを`OperationCanceledException`へ変換し、graph / index / FTS処理より前に全cached statementをfinalizeします。同じtransaction内では3本の`files_resource_generation_*` triggerを停止し、native statementのfinalize後だけ再作成します。fileを1件以上永続化した場合はresource-list generationを厳密に1回進め、空repositoryでは変更しません。rollback時はschemaとgenerationを一括で元へ戻します。`RETURNING` leaseは正のIDとinput ordinalを終端`DONE`まで全件buffer / validationしてから公開し、不正、欠落、重複、失敗、cancelされたstreamではprepared statementを破棄し、data rollbackはcallerのfile単位SAVEPOINTが所有します。replacement、incremental、rebuild、symbols-only、fresh-claim race fallback、MCP、public writer経路はMicrosoft.Data.Sqliteとmutationごとのgeneration invalidationを維持します。 | +| authoritative-fresh raw insert scope | empty-database CLI経路がcaller-owned transaction内でauthoritative-fresh claimを再検証した後に限り、extraction pipelineのnew-file、chunk、symbol、new-file issue、fresh reference-line、atomic fresh-reference INSERTをprovider所有connection handle上のSQLitePCLRawでbind / executeします。native positional bindingは専用の512 parameter上限を使い、provider経由のcaller-owned writeは32 parameter上限を維持します。fresh file / reference-line writeもDONE-onlyです。file insertは同じconnectionの正の`last_insert_rowid`を取得し、reference-line batchは毎回`MAX(id)`と`sqlite_sequence.seq`の大きい方を読み、Int64範囲全体を検証して`?1 + input_ordinal`の明示的な連続IDを挿入します。batchごとのfloor読取により、rollback依存のallocator stateを保持せずAUTOINCREMENT履歴とbatch間insertを反映します。不正floorとidentity range overflowはINSERT実行前に失敗します。実行済みfresh identity writeはID公開前に`sqlite3_changes()`を検証し、row count不一致、constraint、cleanup failure、cancellationでは対象prepared statementを破棄し、data rollbackはcallerのfile単位SAVEPOINTが所有します。scopeは正確なtail / write-count形状、batch hook、row-skip replay、outer transaction atomicityを維持し、32-entry LRUで繰り返すfull / tail statement形状を保持します。同期的なfull-scan persistence consumerと`DbWriter`のtransaction owner検査により、非thread-safe cacheはsingle-ownerのままです。各leaseはresetとbinding clearを行い、error時は元のstep結果を保持し、SQLite interruptを`OperationCanceledException`へ変換し、graph / index / FTS処理より前に全cached statementをfinalizeします。同じtransaction内では3本の`files_resource_generation_*` triggerを停止し、native statementのfinalize後だけ再作成します。fileを1件以上永続化した場合はresource-list generationを厳密に1回進め、空repositoryでは変更しません。rollback時はschemaとgenerationを一括で元へ戻します。replacement、incremental、rebuild、symbols-only、fresh-claim race fallback、MCP、public writer経路はMicrosoft.Data.Sqlite、既存の`RETURNING`挙動、mutationごとのgeneration invalidationを維持します。 | | authoritative-fresh core secondary index | 同じempty-database CLI transactionがauthoritative-fresh claimを再検証した後、`files`、`chunks`、`file_issues`、`symbols`の言語共通secondary index 22本をpersistence前に停止し、全native INSERT statementのfinalize後かつgraph / readiness queryの開始前に各B-treeを1回だけ構築します。UNIQUE autoindexは維持するため、pathとtable constraintは通常どおり適用されます。fresh-reference insertが相関するfile単位lookupでsource symbolを解決するため、`idx_symbols_file`も維持し、このhot pathがsymbol table全体の反復scanへ退行しないようにします。cancel / failure時はouter rollbackがload前のschemaをatomicに復元し、rebuild、incremental、fresh-claim race fallback、MCP経路はwrite中もindexを維持します。canonical DDLをschema initialization、opportunistic read migration、bulk-load guardで共有し、deferred setと完了DBの契約がずれないようにします。 | | checkpoint | `DbWriter` は outer transaction commit 後に `PRAGMA wal_checkpoint(PASSIVE)` を実行し、SQLite も設定済みの 1000 page threshold を超えると自動 checkpoint する場合があります。どちらの checkpoint path も opportunistic で、active reader は block されず、未 checkpoint の WAL は corruption ではなく期待される状態です。 | | checkpoint result contract | 明示的な `PRAGMA wal_checkpoint(TRUNCATE)` path は reader を実行し、SQLite の `(busy, log, checkpointed)` を含む構造化結果を返します。`busy` が 0 以外、または remaining page が正の場合は、上限付き machine reason を伴う unsuccessful result です。`(0, -1, -1)` は SQLite の非 WAL database に対する成功 no-op です。instance checkpoint、read-only fallback 前の static preflight、query diagnostics、top-level status、nested connection-policy status は同じ結果と count を保持します。raw exception text や path を diagnostics に含めてはいけません。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 955807941e..7e5639cdc5 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -465,7 +465,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result - `IndexCommandRunnerTests.Run_MemoryTrace_ReportsFullScanAndUpdatePhaseBoundaries` keeps the full-scan and file-scoped update memory timelines ordered and complete. The fixture uses one C# file, one Python file, and one TypeScript file to cover the C# prepass plus language-neutral extraction, reference-graph, text-index, finalize, and full-scan commit boundaries without scaling up the repository. When a TypeScript augmentation rebuild owns graph finalization, `text_index` precedes the deferred `reference_graph` sample; ordinary updates retain the graph-before-text order. The stale merged-declaration fixture also pins that attribution when no earlier graph pass was pending. - `IndexCommandRunnerTests.Run_InitialFullIndex_MixedLanguageFixture_*` - owns the empty-database initial full-index performance contract. The normal CI case creates the database outside the source tree and runs the real CLI with fixed parallelism over C#, TypeScript, Python, Java, Go, Rust, C++, and Kotlin. Keep its broad wall-clock limit as a runaway guard rather than a speed target, and preserve the stronger phase, persistence-count, readiness, workspace-check, FTS, and integrity assertions. Its preparation observations require exactly one family-scope resolution per ordinary file and one C# source observation per C# file, proving that persistence does not repeat worker work. Its raw-insert observations must keep every new file on the singleton `RETURNING` shape while allowing chunks, symbols, issues, reference lines, and references to use the authoritative-fresh native parameter budget. `AuthoritativeFreshRawBulkInsertTests.BatchStatements_PreserveShapesUnicodeNullsInt64AndProviderExclusions` crosses every multi-row boundary, pins the 512-parameter ceiling and tail shapes, and keeps duplicate RETURNING IDs rejected in linear work. The `ManualRepositoryScaleSmoke` variant uses the same contract with a larger fixture and runs only when `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1` is set. + owns the empty-database initial full-index performance contract. The normal CI case creates the database outside the source tree and runs the real CLI with fixed parallelism over C#, TypeScript, Python, Java, Go, Rust, C++, and Kotlin. Keep its broad wall-clock limit as a runaway guard rather than a speed target, and preserve the stronger phase, persistence-count, readiness, workspace-check, FTS, and integrity assertions. Its preparation observations require exactly one family-scope resolution per ordinary file and one C# source observation per C# file, proving that persistence does not repeat worker work. Its raw-insert observations must keep every new file on the singleton DONE/last-rowid shape while allowing chunks, symbols, issues, reference lines, and references to use the authoritative-fresh native parameter budget. `AuthoritativeFreshRawBulkInsertTests.BatchStatements_PreserveShapesUnicodeNullsInt64AndProviderExclusions` crosses every multi-row boundary and pins the 512-parameter ceiling plus the 511-parameter reference-line shape. Pair it with the SQL-plan guard that rejects `RETURNING` and correlated input scans, and with the ID-floor tests for deleted AUTOINCREMENT history, `MAX(id)` precedence, duplicate sequence rows, batch interleaving, Int64 overflow, changed-row mismatches, and savepoint rollback. The `ManualRepositoryScaleSmoke` variant uses the same contract with a larger fixture and runs only when `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1` is set. - `Run_ParallelCSharpSymbolCapCarriesSourceObservationWithoutAppliedFamilyScope`, `Run_FileAboveMaxReferencesPerFile_FullScanAndUpdatePersistReferenceCountExceededIssueOnly_Issue3719`, `Run_FullScan_ConfiguredGeneratedCodePatternsSkipAllSymbolPreparation`, and `PartialCanonicalRepresentative_SerialPreparationRunsOnceAndPersistsHookRebuiltFamilyScope_Issue4914` pin the full-scan preparation handoff at its exceptional boundaries. Require source/scope call counts of 1/0 for a parallel C# symbol cap, 1/1 for a reference cap, and 0/0 for a generated-suppressed C# file. The hook fixture must force serial extraction, call each stage once, and keep all hook-mutated partial symbols on the rebuilt family key. Keep `FamilyScopeApplied` explicit: a carried `null` key may mean a legitimately applied null scope, while the symbol-cap fixture is explicitly unprepared. Preserve the separate Python symbol-cap fixture so this C# handoff coverage does not replace language-neutral cap behavior. - `IndexCommandRunnerTests.CalculateDefaultIndexParallelism_CapsAutomaticWorkersWithoutLoweringSmallHosts` @@ -777,10 +777,10 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result Concurrent read and read-during-write scenarios (WAL mode validation), including the issue #180 bug-catching snapshot-isolation regressions for all three multi-statement reader entry points: (1) `GetStatus` seeds `refs == files * refsPerFile` and asserts every concurrent observation preserves that invariant; (2) `AnalyzeSymbol` seeds one symbol `S` plus matching reference/caller pairs, toggles a second file symmetrically, and asserts `references.Count == callers.Count` across every `inspect`/`analyze_symbol` bundle; (3) `GetRepoMap` seeds a baseline modified timestamp and toggles a newer file, asserting `latest_modified == workspace_latest_modified` across every map call. Each test fails without the DEFERRED-transaction wrap on the matching reader and passes with it. - `PerformanceTests.cs` Bounded CI smoke coverage plus large-scale data benchmarks. `CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` and the allocation budget guards run in the default `net8.0` suite, so they are blocking PR/CI checks on the production target, but their broad budgets are intended to catch only severe indexing/search or allocation regressions rather than act as benchmarks. `ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` uses dense C# private-property receivers and Python imported-type calls to prevent per-candidate full-symbol rescans from returning. `ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` covers dense C# declaration containers and GitHub Actions jobs so name/range ownership resolution stays indexed. `Extraction_DenseDelimitedLists_StayWithinAllocationBudget` covers Python imports, YAML needs, JSON paths, and Fortran procedure lists without temporary split-array growth. `ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` keeps all-language dedupe identities value-based when qualified names are long. Large-scale manual tests remain skip-by-default; run a selected test on the production target with `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter `. - `AuthoritativeFreshRawBulkInsertTests` fixes the raw-scope activation and single-owner transaction guard, exact 32-parameter/tail/result shapes, 32-entry bounded LRU behavior, Unicode including embedded NUL, stable dates, NULL and 64-bit integer bindings, provider-only routing exclusions, constraint row-skip replay, deterministic interrupt rollback, hook-failure cleanup, finalization counts, and immediate provider reuse. Its file/reference-line `RETURNING` cases cover missing, extra, malformed, duplicate, and out-of-order rows, constraint and cancellation after a returned row, statement discard/reprepare, full buffering before ID publication, and per-file savepoint rollback. The resource-generation case inserts multiple files while the authoritative scope is active, requires one generation advance after commit, verifies an empty successful scope does not advance, verifies all three mutation triggers are restored, proves ordinary provider writes advance again, and proves an abandoned scope rolls schema, generation, and rows back together. Scope statistics mark `Completed=true` only after native statements finalize, resource triggers are restored, the conditional generation advance runs, and cancellation checks succeed; the internal completion marker is set only after the reporting hook returns, so a reporting-hook exception still makes `Complete()` fail and leaves the scope non-completed. Keep the direct tests paired with fresh/rebuild, update, claim-race fallback, MCP, mixed-language CLI, and recoverable file-failure integration assertions so only the authoritative empty CLI extraction interval observes raw work. + `AuthoritativeFreshRawBulkInsertTests` fixes the raw-scope activation and single-owner transaction guard, exact 512-parameter/tail/write-count shapes, 32-entry bounded LRU behavior, Unicode including embedded NUL, stable dates, NULL and 64-bit integer bindings, provider-only routing exclusions, constraint row-skip replay, deterministic interrupt rollback, hook-failure cleanup, finalization counts, and immediate provider reuse. Its fresh identity cases cover positive 64-bit file last-rowids, reference-line floors from both the live table and deleted AUTOINCREMENT history, duplicate-sequence fail-closed behavior, the 170/171-row boundary, inserts between batches, Int64 exhaustion before DML, changed-row mismatches, constraint/cancellation discard and reprepare, ID publication only after successful DML, and per-file savepoint rollback. The raw SQL-plan guard requires one explicit contiguous-ID INSERT without `RETURNING` or a correlated input scan; provider, rebuild, incremental, and MCP coverage retain their established `RETURNING` path. The resource-generation case inserts multiple files while the authoritative scope is active, requires one generation advance after commit, verifies an empty successful scope does not advance, verifies all three mutation triggers are restored, proves ordinary provider writes advance again, and proves an abandoned scope rolls schema, generation, and rows back together. Scope statistics mark `Completed=true` only after native statements finalize, resource triggers are restored, the conditional generation advance runs, and cancellation checks succeed; the internal completion marker is set only after the reporting hook returns, so a reporting-hook exception still makes `Complete()` fail and leaves the scope non-completed. Keep the direct tests paired with fresh/rebuild, update, claim-race fallback, MCP, mixed-language CLI, and recoverable file-failure integration assertions so only the authoritative empty CLI extraction interval observes raw work. `PreparedCommandCacheTests.DbWriter_WithCache_FixedSavepointControlsReuseAcrossCommitRollbackAndTransactionRebind` pins cache misses on the first depth-one SAVEPOINT / RELEASE / ROLLBACK and cache hits after the outer transaction changes. Pair it with the deep-savepoint test, which keeps depth two and beyond outside the cache, the cancelled-nested-begin test, which leases no control command and leaves the outer scope reusable, and the metadata/FTS test, which applies the same fixed-statement contract across every atomic marker surface. `CoreSecondaryIndexBulkLoadGuardTests` pins the exact canonical set of 23 language-neutral `files` / `chunks` / `file_issues` / `symbols` secondary indexes and the 22-index deferred subset, requires a caller-owned transaction at both scope boundaries, keeps the file-path UNIQUE autoindex and reference-source `idx_symbols_file` lookup active, exercises restore over a populated table, and proves cancellation leaves DDL recovery to outer rollback. The full-scan reference-index lifecycle theory pairs this direct coverage with production ordering: only the authoritative fresh CLI run observes `dropped` then `restored`, the raw scope has finalized before restore, every graph phase sees the complete core set, and a nonempty `--rebuild` never defers it. - `PerformanceTests.AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations` alternates three measured provider/raw pairs after one warm-up over one file and 10,000 chunks, symbols, issues, and references, using the production 32-parameter boundaries and each side's own file/reference-line `RETURNING` path. It asserts exact persisted-result parity, including file timestamp text, and reports per-stage/total elapsed time plus current-thread allocation without a wall-clock ratio gate. Run it with `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter FullyQualifiedName~AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations`. + `PerformanceTests.AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations` alternates three measured provider/raw pairs after one warm-up over one file and 10,000 chunks, symbols, issues, and references, using the production 32-parameter provider boundaries, provider `RETURNING`, and raw DONE/explicit-identity paths. It asserts exact persisted-result parity, including file timestamp text, and reports per-stage/total elapsed time plus current-thread allocation without a wall-clock ratio gate. Run it with `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter FullyQualifiedName~AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations`. Focused authoritative-fresh fold-readiness coverage spans `DatabaseTests`, `IndexCommandRunnerTests`, and `McpServerToolsCallTests`: the built-in empty-database CLI/MCP path must consume its claim once, retain the NULL-column verification, skip the stored-value re-fold scan, and produce the same readiness/version/fingerprint/language stamps and Unicode, Markdown, C#, Nim, and TypeScript query results as full validation. Pair it with fail-closed cases for each initially nonempty ownership table (`files`, `symbols`, or `symbol_references`), a wrong or reused claim, an intervening external commit observed through `PRAGMA data_version`, rebuild/update/legacy/public-writer paths, and custom plugins, patterns, or post-extraction hooks; full validation must still reject NULL and stale non-NULL folds. A run-barrier regression must also activate a custom producer and then reload back to built-in-only before readiness: the current producer count returns to zero, but the monotonic mutation generation changes and forces full validation. Unchanged missing-directory and diagnostic-only publications must not change that generation. A deterministic cancel-after-`BEGIN IMMEDIATE` test must prove that the raw transaction is rolled back and the same writer can immediately start and commit another transaction. For performance audits, alternate identical repository-scale fresh fixtures, isolate the readiness-finalization interval, and report elapsed time plus `GC.GetAllocatedBytesForCurrentThread`; adoption requires removing row-count-proportional managed allocation without changing rows, stamps, or query results. Keep wall-clock measurements out of blocking CI assertions and remove temporary instrumentation after recording the result. @@ -1930,9 +1930,9 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" 並行読み取りと書き込み中読み取りシナリオ（WALモード検証）。issue #180 の bug-catching な snapshot 隔離回帰テストを 3 つの multi-statement reader 経路について含む。(1) `GetStatus` は `refs == files * refsPerFile` の seed 不変条件を立て、並行観測が常にこの条件を維持することを要求する。(2) `AnalyzeSymbol` はシンボル `S` に対して reference/caller を対称に 1 対 1 で seed し、もう 1 ファイルを対称に toggle することで `inspect` / `analyze_symbol` bundle の `references.Count == callers.Count` を常に保証する。(3) `GetRepoMap` はベースラインの modified と新しい toggle 対象ファイルを用意し、`latest_modified == workspace_latest_modified` が常に一致することを要求する。各テストは対応する reader の DEFERRED transaction を外すと落ち、戻すと通ることを確認済み。 - `PerformanceTests.cs` bounded な CI smoke と大規模データベンチマークを扱います。`CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` と allocation budget guard は通常の `net8.0` suite で実行されるため production target 上の PR / CI blocking check ですが、benchmark ではなく重大な indexing/search または allocation 退行だけを拾う広めの budget を使います。`ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` は密な C# private-property receiver と Python imported-type call を使い、candidate ごとの full-symbol 再走査が戻るのを防ぎます。`ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` は密な C# declaration container と GitHub Actions job を扱い、name / range ownership 解決の索引化を維持します。`Extraction_DenseDelimitedLists_StayWithinAllocationBudget` は Python import、YAML needs、JSON path、Fortran procedure list を使い、一時 split-array の増加を防ぎます。`ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` は長い qualified name でも全言語共通 dedupe identity を value-based に維持します。大規模な手動 test は引き続きデフォルト Skip とし、production target で `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter ` を実行します。 - `AuthoritativeFreshRawBulkInsertTests` はraw scopeのactivationとsingle-owner transaction guard、正確な32 parameter / tail / result形状、32-entry bounded LRU、embedded NULを含むUnicode、stable date、NULL / 64-bit integer binding、provider専用routing除外、constraint時のrow-skip replay、deterministic interrupt rollback、hook failure cleanup、finalize count、直後のprovider再利用を固定します。file / reference-lineの`RETURNING` caseはrowの欠落、過剰、不正、重複、順序変更、row返却後のconstraint / cancellation、statement discard / reprepare、ID公開前の全件buffer、file単位SAVEPOINT rollbackを検証します。scope statsの`Completed=true`はnative statementのfinalizeとcancellation check成功後を示しますが、内部completion markerはreporting hookの正常return後にだけ設定するため、reporting-hook exceptionでは`Complete()`が失敗しscopeもnon-completedのままです。direct testをfresh / rebuild、update、claim-race fallback、MCP、mixed-language CLI、recoverable file failureのintegration assertionと対にし、authoritative empty CLI extraction区間だけがraw workを観測することを維持してください。 + `AuthoritativeFreshRawBulkInsertTests` はraw scopeのactivationとsingle-owner transaction guard、正確な512 parameter / tail / write-count形状、32-entry bounded LRU、embedded NULを含むUnicode、stable date、NULL / 64-bit integer binding、provider専用routing除外、constraint時のrow-skip replay、deterministic interrupt rollback、hook failure cleanup、finalize count、直後のprovider再利用を固定します。fresh identity caseは正の64-bit file last-rowid、live tableと削除済みAUTOINCREMENT履歴の両方から得るreference-line floor、重複sequence rowのfail-closed、170/171 row境界、batch間insert、DML前のInt64枯渇、changed-row不一致、constraint / cancellation時のdiscard / reprepare、DML成功後だけのID公開、file単位SAVEPOINT rollbackを検証します。raw SQL-plan guardは`RETURNING`や相関input scanを含まない明示的な連続ID INSERTを要求し、provider、rebuild、incremental、MCP coverageは既存の`RETURNING`経路を維持します。scope statsの`Completed=true`はnative statementのfinalizeとcancellation check成功後を示しますが、内部completion markerはreporting hookの正常return後にだけ設定するため、reporting-hook exceptionでは`Complete()`が失敗しscopeもnon-completedのままです。direct testをfresh / rebuild、update、claim-race fallback、MCP、mixed-language CLI、recoverable file failureのintegration assertionと対にし、authoritative empty CLI extraction区間だけがraw workを観測することを維持してください。 `PreparedCommandCacheTests.DbWriter_WithCache_FixedSavepointControlsReuseAcrossCommitRollbackAndTransactionRebind` は、depth 1 の SAVEPOINT / RELEASE / ROLLBACK が初回だけ cache miss となり、outer transaction が変わった後は cache hit となることを固定します。depth 2 以深を cache 外に保つ deep-savepoint test、control command を借りず outer scope を再利用可能なままにする cancelled-nested-begin test、同じ固定 statement 契約をすべての atomic marker surface へ横展開する metadata / FTS test と対にしてください。 - `PerformanceTests.AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations` は1 fileと10,000件ずつのchunk、symbol、issue、referenceについて、1回のwarm-up後にprovider / rawの3 measured pairを交互に実行します。productionの32 parameter境界と各側固有のfile / reference-line `RETURNING`経路を使い、file timestamp textを含む永続化結果の完全一致をassertし、wall-clock ratio gateを設けずstage別 / totalの経過時間とcurrent-thread allocationを報告します。`CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter FullyQualifiedName~AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations`で実行してください。 + `PerformanceTests.AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations` は1 fileと10,000件ずつのchunk、symbol、issue、referenceについて、1回のwarm-up後にprovider / rawの3 measured pairを交互に実行します。productionの32 parameter provider境界、provider `RETURNING`、raw DONE / explicit-identity経路を使い、file timestamp textを含む永続化結果の完全一致をassertし、wall-clock ratio gateを設けずstage別 / totalの経過時間とcurrent-thread allocationを報告します。`CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter FullyQualifiedName~AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations`で実行してください。 authoritative-fresh fold readiness の focused coverage は `DatabaseTests`、`IndexCommandRunnerTests`、`McpServerToolsCallTests` で分担します。built-in の empty-database CLI / MCP 経路が claim を一度だけ consume し、NULL column の検証を維持しつつ、保存 value の再 fold scan を省き、full validation と同じ readiness / version / fingerprint / language stamp、および Unicode、Markdown、C#、Nim、TypeScript の query result を生成することを固定します。初期状態で ownership table（`files`、`symbols`、`symbol_references`）のいずれかが非空の場合、owner が異なるか再利用された claim、`PRAGMA data_version` で観測される外部 commit、rebuild / update / legacy / public-writer 経路、custom plugin / pattern / post-extraction hook は fail closed であることも対にし、full validation が NULL と stale な非 NULL fold を引き続き拒否することを確認します。run barrier では custom producer を一度 active にしてから readiness 前に built-in-only へ reload し、最終 producer count が zero に戻っていても monotonic mutation generation の変化で full validation へ戻ることを固定します。状態不変の missing-directory と diagnostic-only publication では generation が変わらないことも確認します。 `BEGIN IMMEDIATE`成功直後のdeterministicなcancel testでは、raw transactionがrollbackされ、同じwriterが直後に別transactionを開始・commitできることを必須とします。 性能監査では、同一の repository-scale fresh fixture を交互に実行し、readiness finalization 区間を分離して、経過時間と `GC.GetAllocatedBytesForCurrentThread` を報告します。row 数に比例する managed allocation を取り除きつつ、row、stamp、query result が変わらないことを採用条件にします。wall-clock 計測は blocking CI assertion にせず、結果を記録したら一時 instrumentation を削除してください。 diff --git a/changelog.d/unreleased/+initial-full-index-fresh-reference-line-ids.changed.md b/changelog.d/unreleased/+initial-full-index-fresh-reference-line-ids.changed.md new file mode 100644 index 0000000000..2f7118868d --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-fresh-reference-line-ids.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.AuthoritativeFreshIdentityInsert.cs + - src/CodeIndex/Database/DbWriter.ReferenceSql.cs +--- + +## English + +- **Cold reference-line inserts avoid correlated `RETURNING` scans** — The authoritative empty-database path now assigns checked contiguous IDs from the live table and AUTOINCREMENT history, validates each DONE statement's changed-row count, and publishes IDs only after success. Provider, rebuild, incremental, and MCP paths retain their established behavior. + +## 日本語 + +- **初回 reference-line insert から相関 `RETURNING` scan を削減** — authoritative な空DB経路は live table と AUTOINCREMENT 履歴から検証済みの連続IDを割り当て、DONE statementごとの変更行数を確認し、成功後だけIDを公開します。provider、rebuild、incremental、MCP経路の既存挙動は維持します。 diff --git a/src/CodeIndex/Database/DbWriter.AuthoritativeFreshBulkInsert.cs b/src/CodeIndex/Database/DbWriter.AuthoritativeFreshBulkInsert.cs index 12fe64d442..df4b217ccf 100644 --- a/src/CodeIndex/Database/DbWriter.AuthoritativeFreshBulkInsert.cs +++ b/src/CodeIndex/Database/DbWriter.AuthoritativeFreshBulkInsert.cs @@ -15,10 +15,8 @@ private static readonly AsyncLocal ScopedAuthoritativeFreshRawInsertScopeDisposedForTesting = new(); private static readonly AsyncLocal ScopedAuthoritativeFreshRawStatementCacheCapacityForTesting = new(); - private static readonly AsyncLocal?> - ScopedAuthoritativeFreshRawReturningRowForTesting = new(); - private static readonly AsyncLocal?> - ScopedAuthoritativeFreshRawReturningSqlForTesting = new(); + private static readonly AsyncLocal?> + ScopedAuthoritativeFreshRawChangedRowCountForTesting = new(); private AuthoritativeFreshBulkInsertScope? _authoritativeFreshBulkInsertScope; internal sealed record AuthoritativeFreshRawInsertWork( @@ -39,17 +37,10 @@ internal sealed record AuthoritativeFreshRawInsertScopeStats( long FinalizeCount, bool Completed); - internal readonly record struct AuthoritativeFreshRawReturningRow( + internal readonly record struct AuthoritativeFreshRawChangedRowCount( string Operation, int StatementRows, - int ResultIndex, - long Id, - int? InputOrdinal); - - internal sealed record AuthoritativeFreshRawReturningSql( - string Operation, - int StatementRows, - string Sql); + int ActualChangedRows); internal static Action? AuthoritativeFreshRawInsertExecutingForTesting @@ -71,18 +62,11 @@ internal static int? AuthoritativeFreshRawStatementCacheCapacityForTesting set => ScopedAuthoritativeFreshRawStatementCacheCapacityForTesting.Value = value; } - internal static Func? - AuthoritativeFreshRawReturningRowForTesting + internal static Func? + AuthoritativeFreshRawChangedRowCountForTesting { - get => ScopedAuthoritativeFreshRawReturningRowForTesting.Value; - set => ScopedAuthoritativeFreshRawReturningRowForTesting.Value = value; - } - - internal static Func? - AuthoritativeFreshRawReturningSqlForTesting - { - get => ScopedAuthoritativeFreshRawReturningSqlForTesting.Value; - set => ScopedAuthoritativeFreshRawReturningSqlForTesting.Value = value; + get => ScopedAuthoritativeFreshRawChangedRowCountForTesting.Value; + set => ScopedAuthoritativeFreshRawChangedRowCountForTesting.Value = value; } internal AuthoritativeFreshBulkInsertScope? BeginAuthoritativeFreshBulkInsertScope( @@ -146,6 +130,7 @@ private enum AuthoritativeFreshRawInsertKind Chunks, Symbols, Issues, + ReferenceLineIdFloor, ReferenceLines, References, } @@ -770,7 +755,10 @@ internal void BindDateTimeText(DateTime value) bytes[..byteCount])); } - internal void ExecuteDone() + internal long? ExecuteDone( + string? operation = null, + int? expectedChangedRows = null, + bool captureLastInsertRowId = false) { if (_boundParameterCount != _cached.ParameterCount) { @@ -778,9 +766,18 @@ internal void ExecuteDone() "Raw SQLite binding did not fill the prepared statement " + $"(expected={_cached.ParameterCount}, actual={_boundParameterCount})."); } + if (expectedChangedRows.HasValue + && (expectedChangedRows.Value <= 0 || string.IsNullOrWhiteSpace(operation))) + { + throw new ArgumentException( + "Raw SQLite changed-row validation requires an operation and a positive expected count."); + } + if (captureLastInsertRowId && !expectedChangedRows.HasValue) + throw new ArgumentException("Last-insert ID capture requires changed-row validation."); _scope._cancellationToken.ThrowIfCancellationRequested(); var stepResult = raw.sqlite3_step(_cached.Statement); + long? lastInsertRowId = null; Exception? executionFailure = stepResult switch { raw.SQLITE_DONE => null, @@ -788,6 +785,39 @@ internal void ExecuteDone() "A raw SQLite INSERT unexpectedly returned a row."), _ => _scope.CreateExecutionException(stepResult), }; + if (executionFailure == null && expectedChangedRows is { } expected) + { + try + { + var actual = raw.sqlite3_changes(_scope._database); + if (AuthoritativeFreshRawChangedRowCountForTesting is { } transform) + { + actual = transform(new AuthoritativeFreshRawChangedRowCount( + operation!, + expected, + actual)); + } + if (actual != expected) + { + executionFailure = new InvalidDataException( + $"Raw SQLite {operation} changed an unexpected number of rows " + + $"(expected={expected}, actual={actual})."); + } + } + catch (Exception exception) + { + executionFailure = exception; + } + } + if (executionFailure == null && captureLastInsertRowId) + { + lastInsertRowId = raw.sqlite3_last_insert_rowid(_scope._database); + if (lastInsertRowId <= 0) + { + executionFailure = new InvalidDataException( + $"Raw SQLite {operation} produced a non-positive last insert ID {lastInsertRowId}."); + } + } var resetResult = raw.sqlite3_reset(_cached.Statement); Exception? cleanupFailure = null; @@ -800,8 +830,14 @@ internal void ExecuteDone() if (clearResult != raw.SQLITE_OK && cleanupFailure == null) cleanupFailure = _scope.CreateExecutionException(clearResult); - if (!resetIsReusable || clearResult != raw.SQLITE_OK) + var cancellationPending = _scope._cancellationToken.IsCancellationRequested; + if (!resetIsReusable + || clearResult != raw.SQLITE_OK + || (expectedChangedRows.HasValue + && (executionFailure != null || cleanupFailure != null || cancellationPending))) + { _scope.DiscardStatement(_cached); + } _cleaned = true; if (executionFailure != null) @@ -809,13 +845,10 @@ internal void ExecuteDone() if (cleanupFailure != null) throw cleanupFailure; _scope._cancellationToken.ThrowIfCancellationRequested(); + return lastInsertRowId; } - internal void ExecuteReturningRows( - string operation, - int expectedRowCount, - Span idsByInputOrdinal, - bool returnsInputOrdinal) + internal long ExecuteInt64Scalar(string operation) { if (_boundParameterCount != _cached.ParameterCount) { @@ -823,121 +856,33 @@ internal void ExecuteReturningRows( "Raw SQLite binding did not fill the prepared statement " + $"(expected={_cached.ParameterCount}, actual={_boundParameterCount})."); } - if (expectedRowCount <= 0) - throw new ArgumentOutOfRangeException(nameof(expectedRowCount)); - if (idsByInputOrdinal.Length != expectedRowCount) - throw new ArgumentException("The raw RETURNING ID buffer must match the expected row count.", nameof(idsByInputOrdinal)); + if (string.IsNullOrWhiteSpace(operation)) + throw new ArgumentException("A raw SQLite scalar operation is required.", nameof(operation)); _scope._cancellationToken.ThrowIfCancellationRequested(); Exception? executionFailure = null; - var returnedRowCount = 0; var terminalResult = raw.SQLITE_OK; - var returningRowTransform = AuthoritativeFreshRawReturningRowForTesting; - HashSet? returnedIds = expectedRowCount > 1 - ? new HashSet(expectedRowCount) - : null; + var value = 0L; try { - while (true) + terminalResult = raw.sqlite3_step(_cached.Statement); + if (terminalResult != raw.SQLITE_ROW) { - terminalResult = raw.sqlite3_step(_cached.Statement); - if (terminalResult == raw.SQLITE_ROW) - { - if (returnedRowCount >= expectedRowCount) - { - throw new InvalidDataException( - $"Raw SQLite RETURNING produced more than {expectedRowCount} rows."); - } - - if (raw.sqlite3_column_type(_cached.Statement, 0) != raw.SQLITE_INTEGER) - { - throw new InvalidDataException( - $"Raw SQLite {operation} RETURNING produced a non-integer ID."); - } - var id = raw.sqlite3_column_int64(_cached.Statement, 0); - int? inputOrdinal = null; - if (returnsInputOrdinal) - { - var ordinalType = raw.sqlite3_column_type(_cached.Statement, 1); - if (ordinalType == raw.SQLITE_INTEGER) - { - var ordinalValue = raw.sqlite3_column_int64(_cached.Statement, 1); - if (ordinalValue is >= int.MinValue and <= int.MaxValue) - inputOrdinal = (int)ordinalValue; - } - else if (ordinalType != raw.SQLITE_NULL) - { - throw new InvalidDataException( - $"Raw SQLite {operation} RETURNING produced a non-integer input ordinal."); - } - } - - if (returningRowTransform is { } transform) - { - var returnedRow = transform(new AuthoritativeFreshRawReturningRow( - operation, - expectedRowCount, - returnedRowCount, - id, - inputOrdinal)); - id = returnedRow.Id; - inputOrdinal = returnedRow.InputOrdinal; - } - - var resolvedOrdinal = returnsInputOrdinal - ? inputOrdinal - : returnedRowCount; - if (id <= 0) - { - throw new InvalidDataException( - $"Raw SQLite {operation} RETURNING produced a non-positive ID."); - } - if (resolvedOrdinal is not { } ordinal - || (uint)ordinal >= (uint)expectedRowCount) - { - throw new InvalidDataException( - $"Raw SQLite {operation} RETURNING produced invalid input ordinal " - + $"{resolvedOrdinal?.ToString(CultureInfo.InvariantCulture) ?? "NULL"} " - + $"for {expectedRowCount} rows."); - } - if (idsByInputOrdinal[ordinal] != 0) - { - throw new InvalidDataException( - $"Raw SQLite {operation} RETURNING produced duplicate input ordinal {ordinal}."); - } - if (returnedIds != null && !returnedIds.Add(id)) - { - throw new InvalidDataException( - $"Raw SQLite {operation} RETURNING produced duplicate ID {id}."); - } - idsByInputOrdinal[ordinal] = id; - returnedRowCount++; - // Do not throw only from the managed token between ROWs. A pending - // sqlite3_interrupt must reach the next step so SQLite preserves its - // transaction rollback semantics before the cancellation escapes. - // ROW間ではmanaged tokenだけでthrowせず、pending interruptを次stepへ渡す。 - continue; - } - - if (terminalResult != raw.SQLITE_DONE) - throw _scope.CreateExecutionException(terminalResult); - if (returnedRowCount != expectedRowCount) - { - throw new InvalidDataException( - "Raw SQLite RETURNING produced an incomplete result " - + $"(expected={expectedRowCount}, actual={returnedRowCount})."); - } - break; + if (terminalResult == raw.SQLITE_DONE) + throw new InvalidDataException($"Raw SQLite {operation} produced no scalar row."); + throw _scope.CreateExecutionException(terminalResult); } - - for (var ordinal = 0; ordinal < idsByInputOrdinal.Length; ordinal++) + if (raw.sqlite3_column_type(_cached.Statement, 0) != raw.SQLITE_INTEGER) { - if (idsByInputOrdinal[ordinal] == 0) - { - throw new InvalidDataException( - $"Raw SQLite {operation} RETURNING did not materialize input ordinal {ordinal}."); - } + throw new InvalidDataException( + $"Raw SQLite {operation} produced a non-integer scalar."); } + value = raw.sqlite3_column_int64(_cached.Statement, 0); + terminalResult = raw.sqlite3_step(_cached.Statement); + if (terminalResult == raw.SQLITE_ROW) + throw new InvalidDataException($"Raw SQLite {operation} produced more than one scalar row."); + if (terminalResult != raw.SQLITE_DONE) + throw _scope.CreateExecutionException(terminalResult); } catch (Exception exception) { @@ -961,10 +906,10 @@ internal void ExecuteReturningRows( || cleanupFailure != null || cancellationPending) { - // SQLite applies a DML RETURNING statement before yielding its first ROW. - // Reset is cleanup, not rollback. Never reuse a statement after any ROW - // protocol failure; the caller's per-file SAVEPOINT owns data rollback. - // DML RETURNINGは最初のROW前に適用済み。resetはrollbackではない。 + // A scalar ROW protocol failure can leave the native statement at an + // uncertain step boundary. Reset is cleanup, not recovery, so reprepare. + // scalar ROW protocol failure後のstep境界は不確実になり得る。 + // resetはcleanupであってrecoveryではないため再prepareする。 _scope.DiscardStatement(_cached); } _cleaned = true; @@ -974,6 +919,7 @@ internal void ExecuteReturningRows( if (cleanupFailure != null) throw cleanupFailure; _scope._cancellationToken.ThrowIfCancellationRequested(); + return value; } internal void Discard() diff --git a/src/CodeIndex/Database/DbWriter.AuthoritativeFreshReturningInsert.cs b/src/CodeIndex/Database/DbWriter.AuthoritativeFreshIdentityInsert.cs similarity index 58% rename from src/CodeIndex/Database/DbWriter.AuthoritativeFreshReturningInsert.cs rename to src/CodeIndex/Database/DbWriter.AuthoritativeFreshIdentityInsert.cs index dd26cd9641..689f90234c 100644 --- a/src/CodeIndex/Database/DbWriter.AuthoritativeFreshReturningInsert.cs +++ b/src/CodeIndex/Database/DbWriter.AuthoritativeFreshIdentityInsert.cs @@ -5,8 +5,19 @@ public partial class DbWriter private const string AuthoritativeFreshRawFileInsertSql = """ INSERT INTO files (path, lang, size, lines, checksum, modified, generated, indexed_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, CURRENT_TIMESTAMP) - RETURNING id """; + private const string AuthoritativeFreshReferenceLineIdFloorSql = """ + SELECT CASE + WHEN (SELECT COUNT(*) FROM sqlite_sequence WHERE name = 'reference_lines') > 1 + THEN -1 + ELSE MAX( + COALESCE((SELECT MAX(id) FROM reference_lines), 0), + COALESCE((SELECT MAX(seq) FROM sqlite_sequence WHERE name = 'reference_lines'), 0)) + END + """; + + internal static string AuthoritativeFreshReferenceLineIdFloorSqlForTesting + => AuthoritativeFreshReferenceLineIdFloorSql; internal sealed partial class AuthoritativeFreshBulkInsertScope { @@ -15,18 +26,11 @@ internal long InsertFile(CodeIndex.Models.FileRecord file) ArgumentNullException.ThrowIfNull(file); EnsureCanExecute(); using var interrupt = _writer.RegisterSqliteInterrupt(_cancellationToken); - var sql = TransformReturningSqlForTesting( - "insert_files", - statementRows: 1, - AuthoritativeFreshRawFileInsertSql); var lease = RentStatementLease( AuthoritativeFreshRawInsertKind.Files, rows: 1, - sql, - expectedParameterCount: 7, - expectedColumnCount: 1); - Span returnedIds = stackalloc long[1]; - returnedIds.Clear(); + AuthoritativeFreshRawFileInsertSql, + expectedParameterCount: 7); try { lease.BindText(file.Path); @@ -38,12 +42,17 @@ internal long InsertFile(CodeIndex.Models.FileRecord file) lease.BindInt64(file.Generated ? 1 : 0); ReportStatementExecution("insert_files", rows: 1, lease); - lease.ExecuteReturningRows( + var fileId = lease.ExecuteDone( "insert_files", - expectedRowCount: 1, - returnedIds, - returnsInputOrdinal: false); - return returnedIds[0]; + expectedChangedRows: 1, + captureLastInsertRowId: true) + ?? throw new InvalidDataException( + "Raw SQLite insert_files did not capture a last insert ID."); + // The fresh scope suspends the only files INSERT trigger and owns the + // connection synchronously, so no intervening INSERT can replace this ID. + // fresh scopeは唯一のfiles INSERT triggerを停止しconnectionを同期所有するため、 + // このIDが別INSERTで置き換わる余地はない。 + return fileId; } finally { @@ -61,32 +70,31 @@ internal void InsertReferenceLines( { EnsureCanExecute(); var statementRows = end - start; - var maximumRows = GetRowsPerAuthoritativeFreshRawInsertStatement(columnCount: 3); + var maximumRows = GetRowsPerAuthoritativeFreshRawInsertStatement( + columnCount: 3, + fixedParameterCount: 1); if (statementRows <= 0 || statementRows > maximumRows) { throw new InvalidOperationException( - "Raw reference-line RETURNING requires the authoritative fresh parameter budget " + "Raw reference-line insert requires the authoritative fresh parameter budget " + $"(rows={statementRows}, maximum={maximumRows})."); } using var interrupt = _writer.RegisterSqliteInterrupt(_cancellationToken); - var baseSql = ReferenceLineInsertSqlCache.GetOrAdd( - statementRows, - static count => BuildReferenceLineInsertSql(count)); - var sql = TransformReturningSqlForTesting( - "insert_reference_lines", + var idFloor = ReadReferenceLineIdFloor(); + var firstId = checked(idFloor + 1); + _ = checked(firstId + statementRows - 1); + var sql = AuthoritativeFreshReferenceLineInsertSqlCache.GetOrAdd( statementRows, - baseSql); + static count => BuildAuthoritativeFreshReferenceLineInsertSql(count)); var lease = RentStatementLease( AuthoritativeFreshRawInsertKind.ReferenceLines, statementRows, sql, - expectedParameterCount: statementRows * 3, - expectedColumnCount: 2); - Span returnedIds = stackalloc long[statementRows]; - returnedIds.Clear(); + expectedParameterCount: checked(statementRows * 3 + 1)); try { + lease.BindInt64(firstId); for (var row = start; row < end; row++) { var (fileId, line, context) = rows[row]; @@ -100,11 +108,7 @@ internal void InsertReferenceLines( "insert_reference_lines", statementRows, statementRows); - lease.ExecuteReturningRows( - "insert_reference_lines", - statementRows, - returnedIds, - returnsInputOrdinal: true); + lease.ExecuteDone("insert_reference_lines", statementRows); try { @@ -119,7 +123,7 @@ internal void InsertReferenceLines( var rowIndex = checked(start + inputOrdinal); var lineOrdinal = rowOrdinals[rowIndex]; var key = rows[rowIndex]; - var id = returnedIds[inputOrdinal]; + var id = checked(firstId + inputOrdinal); lineIds.SetReferenceLineId(lineOrdinal, id); knownLineIds.Add(key, id); } @@ -136,20 +140,30 @@ internal void InsertReferenceLines( } } - private static string TransformReturningSqlForTesting( - string operation, - int statementRows, - string sql) + private long ReadReferenceLineIdFloor() { - if (AuthoritativeFreshRawReturningSqlForTesting is not { } transform) - return sql; - - return transform(new AuthoritativeFreshRawReturningSql( - operation, - statementRows, - sql)) - ?? throw new InvalidDataException( - $"Raw SQLite {operation} RETURNING test hook produced no SQL."); + var lease = RentStatementLease( + AuthoritativeFreshRawInsertKind.ReferenceLineIdFloor, + rows: 1, + AuthoritativeFreshReferenceLineIdFloorSql, + expectedParameterCount: 0, + expectedColumnCount: 1); + try + { + ReportStatementExecution("read_reference_line_id_floor", rows: 1, lease); + var idFloor = lease.ExecuteInt64Scalar("read_reference_line_id_floor"); + if (idFloor < 0) + { + lease.Discard(); + throw new InvalidDataException( + $"Raw SQLite reference-line ID floor was negative ({idFloor})."); + } + return idFloor; + } + finally + { + lease.Dispose(); + } } } } diff --git a/src/CodeIndex/Database/DbWriter.BatchSql.cs b/src/CodeIndex/Database/DbWriter.BatchSql.cs index 547d9f45da..00fab8ef7a 100644 --- a/src/CodeIndex/Database/DbWriter.BatchSql.cs +++ b/src/CodeIndex/Database/DbWriter.BatchSql.cs @@ -104,13 +104,17 @@ private static int GetRowsPerCallerTransactionInsertStatement(int columnCount) MaxCallerTransactionBatchParameters / columnCount)); } - private static int GetRowsPerAuthoritativeFreshRawInsertStatement(int columnCount) + private static int GetRowsPerAuthoritativeFreshRawInsertStatement( + int columnCount, + int fixedParameterCount = 0) { if (columnCount <= 0) throw new ArgumentOutOfRangeException(nameof(columnCount)); + if (fixedParameterCount < 0 || fixedParameterCount >= MaxAuthoritativeFreshRawBatchParameters) + throw new ArgumentOutOfRangeException(nameof(fixedParameterCount)); return Math.Max(1, Math.Min( GetRowsPerInsertStatement(columnCount), - MaxAuthoritativeFreshRawBatchParameters / columnCount)); + (MaxAuthoritativeFreshRawBatchParameters - fixedParameterCount) / columnCount)); } } diff --git a/src/CodeIndex/Database/DbWriter.ReferenceSql.cs b/src/CodeIndex/Database/DbWriter.ReferenceSql.cs index 41fd5f6fe8..9e2a1ccbaf 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceSql.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceSql.cs @@ -172,6 +172,36 @@ FROM input internal static string BuildReferenceLineInsertSqlForTesting(int rowCount) => BuildReferenceLineInsertSql(rowCount); + private static string BuildAuthoritativeFreshReferenceLineInsertSql(int rowCount) + { + var sql = CreateBatchSqlBuilder(rowCount, estimatedCharsPerRow: 72); + sql.Append(@" + WITH input(input_ordinal, file_id, line, context) AS ( + VALUES "); + // ?1 is the checked first ID. The remaining slots preserve row/column order. + var parameterIndex = 1; + for (var row = 0; row < rowCount; row++) + { + if (row > 0) + sql.Append(", "); + sql.Append('(').Append(row); + for (var column = 0; column < 3; column++) + { + sql.Append(", "); + AppendBatchParameter(sql, ref parameterIndex); + } + sql.Append(')'); + } + return sql.Append(@" + ) + INSERT INTO reference_lines (id, file_id, line, context) + SELECT ?1 + input_ordinal, file_id, line, context + FROM input").ToString(); + } + + internal static string BuildAuthoritativeFreshReferenceLineInsertSqlForTesting(int rowCount) + => BuildAuthoritativeFreshReferenceLineInsertSql(rowCount); + private static string BuildReferenceLineValuesInsertSql(int rowCount, string suffix) { var sql = CreateBatchSqlBuilder(rowCount, estimatedCharsPerRow: 64); diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 9b642b8959..a096e54202 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -2687,7 +2687,9 @@ private ReferenceLineBatchMap InsertNewReferenceLines( } int rowsPerStatement = useAuthoritativeFreshRawInsert - ? GetRowsPerAuthoritativeFreshRawInsertStatement(columnCount: 3) + ? GetRowsPerAuthoritativeFreshRawInsertStatement( + columnCount: 3, + fixedParameterCount: 1) : useCallerTransactionParameterBudget ? GetRowsPerCallerTransactionInsertStatement(columnCount: 3) : GetRowsPerInsertStatement(columnCount: 3); diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 118ea4e30a..7ff674c2c6 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -177,6 +177,8 @@ internal static Action? FreshBulkLoadPlannerStatistics private static readonly ConcurrentDictionary ReferenceLineUpsertSqlCache = new(); private static readonly ConcurrentDictionary ReferenceLineLookupSqlCache = new(); private static readonly ConcurrentDictionary ReferenceLineInsertSqlCache = new(); + private static readonly ConcurrentDictionary + AuthoritativeFreshReferenceLineInsertSqlCache = new(); private static readonly BoundedRegex CSharpExternAliasSignatureRegex = new( @"^\s*extern\s+alias\b", RegexOptions.Compiled | RegexOptions.CultureInvariant); diff --git a/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs b/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs index 1888aa5850..df1857e82c 100644 --- a/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs +++ b/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs @@ -183,7 +183,7 @@ public void Scope_CoalescesResourceGenerationAndRestoresTriggersAcrossCommitAndR [Fact] public void BatchStatements_PreserveShapesUnicodeNullsInt64AndProviderExclusions() { - PrimeSequencesForInt64Returning(); + PrimeSequencesForInt64Ids(); var rawWork = new List(); var batchWork = new List(); DbWriter.AuthoritativeFreshRawInsertScopeStats? observedStats = null; @@ -322,7 +322,8 @@ [new ReferenceRecord Assert.Equal([(102, 510), (102, 510), (1, 5)], RowsAndParameters("insert_chunks")); Assert.Equal([(20, 500), (20, 500), (1, 25)], RowsAndParameters("insert_symbols")); Assert.Equal([(85, 510), (85, 510), (1, 6)], RowsAndParameters("insert_issues")); - Assert.Equal([(73, 219)], RowsAndParameters("insert_reference_lines")); + Assert.Equal([(1, 0)], RowsAndParameters("read_reference_line_id_floor")); + Assert.Equal([(73, 220)], RowsAndParameters("insert_reference_lines")); Assert.Equal([(36, 504), (36, 504), (1, 14)], RowsAndParameters("insert_references")); Assert.Contains( batchWork, @@ -334,13 +335,13 @@ [new ReferenceRecord Assert.NotNull(observedStats); Assert.True(observedStats.Completed); Assert.Equal(32, observedStats.Capacity); - Assert.Equal(10, observedStats.PeakCachedStatementCount); - Assert.Equal(14, observedStats.StatementExecutionCount); - Assert.Equal(10, observedStats.PrepareCount); + Assert.Equal(11, observedStats.PeakCachedStatementCount); + Assert.Equal(15, observedStats.StatementExecutionCount); + Assert.Equal(11, observedStats.PrepareCount); Assert.Equal(4, observedStats.CacheHitCount); Assert.Equal(0, observedStats.EvictionCount); Assert.Equal(0, observedStats.DiscardCount); - Assert.Equal(10, observedStats.FinalizeCount); + Assert.Equal(11, observedStats.FinalizeCount); Assert.Equal(1L, ScalarLong("SELECT COUNT(*) FROM files WHERE lang IS NULL")); Assert.Equal(5_000_000_123L, ScalarLong("SELECT size FROM files WHERE path = 'src/raw-shapes.cs'")); @@ -604,44 +605,24 @@ public void Complete_WhenReportingHookThrows_DoesNotMarkScopeCompleted() Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM chunks")); } - [Theory] - [InlineData("callback")] - [InlineData("null_ordinal")] - [InlineData("duplicate_ordinal")] - [InlineData("out_of_range_ordinal")] - [InlineData("duplicate_id")] - public void ReferenceLineReturningFailure_DiscardsStatementAndFileSavepointAllowsNextFile( - string failureMode) + [Fact] + public void ReferenceLineChangedRowCountFailure_DiscardsStatementAndFileSavepointAllowsNextFile() { var rawWork = new List(); - long? firstReturnedId = null; DbWriter.AuthoritativeFreshRawInsertScopeStats? observedStats = null; - var previousRowHook = DbWriter.AuthoritativeFreshRawReturningRowForTesting; + var previousChangedRowsHook = DbWriter.AuthoritativeFreshRawChangedRowCountForTesting; var previousRawHook = DbWriter.AuthoritativeFreshRawInsertExecutingForTesting; var previousStatsHook = DbWriter.AuthoritativeFreshRawInsertScopeDisposedForTesting; + var injected = 0; try { - DbWriter.AuthoritativeFreshRawReturningRowForTesting = row => + DbWriter.AuthoritativeFreshRawChangedRowCountForTesting = change => { - var transformed = previousRowHook?.Invoke(row) ?? row; - if (transformed.Operation != "insert_reference_lines") - return transformed; - if (failureMode == "duplicate_id") - { - if (transformed.ResultIndex == 0) - firstReturnedId = transformed.Id; - else if (transformed.ResultIndex == 1) - return transformed with { Id = firstReturnedId!.Value }; - } - return failureMode switch - { - "callback" => throw new InvalidOperationException("returning row callback failed"), - "null_ordinal" => transformed with { InputOrdinal = null }, - "duplicate_ordinal" when transformed.ResultIndex == 1 - => transformed with { InputOrdinal = 0 }, - "out_of_range_ordinal" => transformed with { InputOrdinal = transformed.StatementRows }, - _ => transformed, - }; + var actual = previousChangedRowsHook?.Invoke(change) ?? change.ActualChangedRows; + return change.Operation == "insert_reference_lines" + && Interlocked.Exchange(ref injected, 1) == 0 + ? actual - 1 + : actual; }; DbWriter.AuthoritativeFreshRawInsertExecutingForTesting = work => { @@ -664,34 +645,19 @@ public void ReferenceLineReturningFailure_DiscardsStatementAndFileSavepointAllow using (var failedFile = _writer.BeginTransaction()) { - var failedFileId = InsertNewFile("src/failed-returning.cs"); - var exception = Record.Exception(() => + var failedFileId = InsertNewFile("src/failed-row-count.cs"); + var exception = Assert.Throws(() => _writer.InsertReferencesForNewFilesInAtomicFileScope( CreateReferences(failedFileId, 2, "failed"), refreshMutualRecursionFlags: false, CancellationToken.None)); - if (failureMode == "callback") - { - var callbackException = Assert.IsType(exception); - Assert.Equal("returning row callback failed", callbackException.Message); - } - else - { - var protocolException = Assert.IsType(exception); - var expectedMessagePart = failureMode switch - { - "duplicate_ordinal" => "duplicate input ordinal", - "duplicate_id" => "duplicate ID", - _ => "invalid input ordinal", - }; - Assert.Contains(expectedMessagePart, protocolException.Message, StringComparison.Ordinal); - } + Assert.Contains("expected=2, actual=1", exception.Message, StringComparison.Ordinal); } - DbWriter.AuthoritativeFreshRawReturningRowForTesting = previousRowHook; + DbWriter.AuthoritativeFreshRawChangedRowCountForTesting = previousChangedRowsHook; using (var succeedingFile = _writer.BeginTransaction()) { - var succeedingFileId = InsertNewFile("src/succeeding-returning.cs"); + var succeedingFileId = InsertNewFile("src/succeeding-row-count.cs"); _writer.InsertReferencesForNewFilesInAtomicFileScope( CreateReferences(succeedingFileId, 2, "succeeding"), refreshMutualRecursionFlags: false, @@ -704,13 +670,13 @@ public void ReferenceLineReturningFailure_DiscardsStatementAndFileSavepointAllow } finally { - DbWriter.AuthoritativeFreshRawReturningRowForTesting = previousRowHook; + DbWriter.AuthoritativeFreshRawChangedRowCountForTesting = previousChangedRowsHook; DbWriter.AuthoritativeFreshRawInsertExecutingForTesting = previousRawHook; DbWriter.AuthoritativeFreshRawInsertScopeDisposedForTesting = previousStatsHook; } - Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM files WHERE path = 'src/failed-returning.cs'")); - Assert.Equal(1L, ScalarLong("SELECT COUNT(*) FROM files WHERE path = 'src/succeeding-returning.cs'")); + Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM files WHERE path = 'src/failed-row-count.cs'")); + Assert.Equal(1L, ScalarLong("SELECT COUNT(*) FROM files WHERE path = 'src/succeeding-row-count.cs'")); Assert.Equal(2L, ScalarLong("SELECT COUNT(*) FROM reference_lines")); Assert.Equal(2L, ScalarLong("SELECT COUNT(*) FROM symbol_references")); var referenceLineWork = rawWork @@ -718,43 +684,36 @@ public void ReferenceLineReturningFailure_DiscardsStatementAndFileSavepointAllow .ToArray(); Assert.Equal(2, referenceLineWork.Length); Assert.All(referenceLineWork, work => Assert.False(work.CacheHit)); + var floorWork = rawWork + .Where(work => work.Operation == "read_reference_line_id_floor") + .ToArray(); + Assert.Equal(2, floorWork.Length); + Assert.False(floorWork[0].CacheHit); + Assert.True(floorWork[1].CacheHit); Assert.NotNull(observedStats); Assert.True(observedStats.Completed); Assert.Equal(1, observedStats.DiscardCount); Assert.Equal(observedStats.PrepareCount, observedStats.FinalizeCount); } - [Theory] - [InlineData("missing")] - [InlineData("extra")] - public void FileReturningRowCountFailure_DiscardsStatementAndRollsBackEveryReturnedFile( - string failureMode) + [Fact] + public void FileChangedRowCountFailure_DiscardsStatementAndRollsBackInsertedFile() { var rawWork = new List(); DbWriter.AuthoritativeFreshRawInsertScopeStats? observedStats = null; - var previousSqlHook = DbWriter.AuthoritativeFreshRawReturningSqlForTesting; + var previousChangedRowsHook = DbWriter.AuthoritativeFreshRawChangedRowCountForTesting; var previousRawHook = DbWriter.AuthoritativeFreshRawInsertExecutingForTesting; var previousStatsHook = DbWriter.AuthoritativeFreshRawInsertScopeDisposedForTesting; + var injected = 0; try { - DbWriter.AuthoritativeFreshRawReturningSqlForTesting = statement => + DbWriter.AuthoritativeFreshRawChangedRowCountForTesting = change => { - if (statement.Operation != "insert_files") - return previousSqlHook?.Invoke(statement) ?? statement.Sql; - return failureMode == "missing" - ? """ - INSERT INTO files (path, lang, size, lines, checksum, modified, generated, indexed_at) - SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, CURRENT_TIMESTAMP - WHERE 0 - RETURNING id - """ - : """ - INSERT INTO files (path, lang, size, lines, checksum, modified, generated, indexed_at) - SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, CURRENT_TIMESTAMP - UNION ALL - SELECT ?1 || '.extra', ?2, ?3, ?4, ?5, ?6, ?7, CURRENT_TIMESTAMP - RETURNING id - """; + var actual = previousChangedRowsHook?.Invoke(change) ?? change.ActualChangedRows; + return change.Operation == "insert_files" + && Interlocked.Exchange(ref injected, 1) == 0 + ? 0 + : actual; }; DbWriter.AuthoritativeFreshRawInsertExecutingForTesting = work => { @@ -776,14 +735,15 @@ RETURNING id CancellationToken.None)!; using (var failedFile = _writer.BeginTransaction()) { - Assert.Throws(() => - InsertNewFile("src/file-row-count-failed.cs")); + var exception = Assert.Throws(() => + InsertNewFile("src/file-change-count-failed.cs")); + Assert.Contains("expected=1, actual=0", exception.Message, StringComparison.Ordinal); } - DbWriter.AuthoritativeFreshRawReturningSqlForTesting = previousSqlHook; + DbWriter.AuthoritativeFreshRawChangedRowCountForTesting = previousChangedRowsHook; using (var succeedingFile = _writer.BeginTransaction()) { - _ = InsertNewFile("src/file-row-count-succeeded.cs"); + _ = InsertNewFile("src/file-change-count-succeeded.cs"); succeedingFile.Commit(); } raw.Complete(); @@ -791,13 +751,13 @@ RETURNING id } finally { - DbWriter.AuthoritativeFreshRawReturningSqlForTesting = previousSqlHook; + DbWriter.AuthoritativeFreshRawChangedRowCountForTesting = previousChangedRowsHook; DbWriter.AuthoritativeFreshRawInsertExecutingForTesting = previousRawHook; DbWriter.AuthoritativeFreshRawInsertScopeDisposedForTesting = previousStatsHook; } - Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM files WHERE path LIKE 'src/file-row-count-failed.cs%'")); - Assert.Equal(1L, ScalarLong("SELECT COUNT(*) FROM files WHERE path = 'src/file-row-count-succeeded.cs'")); + Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM files WHERE path = 'src/file-change-count-failed.cs'")); + Assert.Equal(1L, ScalarLong("SELECT COUNT(*) FROM files WHERE path = 'src/file-change-count-succeeded.cs'")); var fileWork = rawWork.Where(work => work.Operation == "insert_files").ToArray(); Assert.Equal(2, fileWork.Length); Assert.All(fileWork, work => Assert.False(work.CacheHit)); @@ -807,7 +767,7 @@ RETURNING id } [Fact] - public void ReferenceLineReturningConstraint_DiscardsStatementBeforeRowAndCanReprepare() + public void ReferenceLineConstraint_DiscardsStatementAndCanReprepare() { long fileId; using (var seedTransaction = _writer.BeginTransaction()) @@ -869,25 +829,29 @@ public void ReferenceLineReturningConstraint_DiscardsStatementBeforeRowAndCanRep } [Fact] - public void ReferenceLineReturningInterruptAfterFirstRow_PreservesCancellationAndRollsBackOuterTransaction() + public void ReferenceLineInterrupt_PreservesCancellationAndRollsBackOuterTransaction() { using var cancellation = new CancellationTokenSource(); + _db.Connection.CreateFunction( + "cancel_authoritative_fresh_reference_line", + () => + { + cancellation.Cancel(); + return 0; + }); + Execute(""" + CREATE TEMP TRIGGER cancel_authoritative_fresh_reference_line_insert + BEFORE INSERT ON reference_lines + BEGIN + SELECT cancel_authoritative_fresh_reference_line(); + END + """); + DbWriter.AuthoritativeFreshRawInsertScopeStats? observedStats = null; - var previousRowHook = DbWriter.AuthoritativeFreshRawReturningRowForTesting; var previousStatsHook = DbWriter.AuthoritativeFreshRawInsertScopeDisposedForTesting; OperationCanceledException exception; try { - DbWriter.AuthoritativeFreshRawReturningRowForTesting = row => - { - var transformed = previousRowHook?.Invoke(row) ?? row; - if (transformed.Operation == "insert_reference_lines" - && transformed.ResultIndex == 0) - { - cancellation.Cancel(); - } - return transformed; - }; DbWriter.AuthoritativeFreshRawInsertScopeDisposedForTesting = stats => { observedStats = stats; @@ -913,7 +877,6 @@ public void ReferenceLineReturningInterruptAfterFirstRow_PreservesCancellationAn } finally { - DbWriter.AuthoritativeFreshRawReturningRowForTesting = previousRowHook; DbWriter.AuthoritativeFreshRawInsertScopeDisposedForTesting = previousStatsHook; } @@ -926,6 +889,7 @@ public void ReferenceLineReturningInterruptAfterFirstRow_PreservesCancellationAn Assert.False(observedStats.Completed); Assert.Equal(1, observedStats.DiscardCount); + Execute("DROP TRIGGER cancel_authoritative_fresh_reference_line_insert"); using var retry = _writer.BeginTransaction(); var retryFileId = InsertNewFile("src/reference-line-cancel-retry.cs"); _writer.InsertReferencesForNewFilesInAtomicFileScope( @@ -936,6 +900,166 @@ public void ReferenceLineReturningInterruptAfterFirstRow_PreservesCancellationAn Assert.Equal(1L, ScalarLong("SELECT COUNT(*) FROM reference_lines")); } + [Fact] + public void ReferenceLineIds_RecheckFloorAcrossBatchesAndPreferTableMaximum() + { + long seedFileId; + using (var seedTransaction = _writer.BeginTransaction()) + { + seedFileId = InsertNewFile("src/reference-line-floor-seed.cs"); + seedTransaction.Commit(); + } + Execute($""" + INSERT INTO reference_lines (id, file_id, line, context) + VALUES (9000, {seedFileId.ToString(CultureInfo.InvariantCulture)}, 1, 'table-floor'); + UPDATE sqlite_sequence SET seq = 5 WHERE name = 'reference_lines'; + """); + + var previousRawHook = DbWriter.AuthoritativeFreshRawInsertExecutingForTesting; + var rawWork = new List(); + var floorReads = 0; + try + { + DbWriter.AuthoritativeFreshRawInsertExecutingForTesting = work => + { + rawWork.Add(work); + if (work.Operation == "read_reference_line_id_floor" + && Interlocked.Increment(ref floorReads) == 2) + { + Execute($""" + INSERT INTO reference_lines (id, file_id, line, context) + VALUES (10000, {seedFileId.ToString(CultureInfo.InvariantCulture)}, 2, 'between-batches'); + """); + } + previousRawHook?.Invoke(work); + }; + + using var graph = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true); + using var transaction = _writer.BeginTransaction(); + using var raw = _writer.BeginAuthoritativeFreshBulkInsertScope( + enabled: true, + CancellationToken.None)!; + var fileId = InsertNewFile("src/reference-line-floor-batches.cs"); + _writer.InsertReferencesForNewFilesInAtomicFileScope( + CreateReferences(fileId, 171, "floor"), + refreshMutualRecursionFlags: false, + CancellationToken.None); + raw.Complete(); + transaction.Commit(); + + Assert.Equal(9001L, ReferenceLineId(fileId, line: 1, "floor_0();")); + Assert.Equal(9170L, ReferenceLineId(fileId, line: 170, "floor_169();")); + Assert.Equal(10001L, ReferenceLineId(fileId, line: 171, "floor_170();")); + } + finally + { + DbWriter.AuthoritativeFreshRawInsertExecutingForTesting = previousRawHook; + } + + Assert.Equal(2, floorReads); + Assert.Equal( + [(170, 511), (1, 4)], + rawWork + .Where(work => work.Operation == "insert_reference_lines") + .Select(work => (work.StatementRows, work.BoundParameterCount)) + .ToArray()); + Assert.Equal(10001L, ScalarLong("SELECT seq FROM sqlite_sequence WHERE name = 'reference_lines'")); + } + + [Fact] + public void ReferenceLineIdReservation_OverflowFailsBeforeInsertAndRollsBackFileSavepoint() + { + long seedFileId; + using (var seedTransaction = _writer.BeginTransaction()) + { + seedFileId = InsertNewFile("src/reference-line-overflow-seed.cs"); + seedTransaction.Commit(); + } + Execute($""" + INSERT INTO reference_lines (id, file_id, line, context) + VALUES ({long.MaxValue.ToString(CultureInfo.InvariantCulture)}, + {seedFileId.ToString(CultureInfo.InvariantCulture)}, 1, 'overflow-primer'); + DELETE FROM reference_lines WHERE id = {long.MaxValue.ToString(CultureInfo.InvariantCulture)}; + """); + + using var graph = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true); + using var outerTransaction = _writer.BeginTransaction(); + using var raw = _writer.BeginAuthoritativeFreshBulkInsertScope( + enabled: true, + CancellationToken.None)!; + using (var failedFile = _writer.BeginTransaction()) + { + var fileId = InsertNewFile("src/reference-line-overflow.cs"); + Assert.Throws(() => + _writer.InsertReferencesForNewFilesInAtomicFileScope( + CreateReferences(fileId, 1, "overflow"), + refreshMutualRecursionFlags: false, + CancellationToken.None)); + } + raw.Complete(); + outerTransaction.Commit(); + + Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM reference_lines")); + Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM files WHERE path = 'src/reference-line-overflow.cs'")); + } + + [Fact] + public void ReferenceLineIdFloor_DuplicateSequenceRowsFailBeforeInsertAndRollBackFileSavepoint() + { + Execute(""" + INSERT INTO sqlite_sequence (name, seq) + VALUES ('reference_lines', 10), ('reference_lines', 20) + """); + + var rawWork = new List(); + var previousRawHook = DbWriter.AuthoritativeFreshRawInsertExecutingForTesting; + try + { + DbWriter.AuthoritativeFreshRawInsertExecutingForTesting = work => + { + rawWork.Add(work); + previousRawHook?.Invoke(work); + }; + + using var graph = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true); + using var outerTransaction = _writer.BeginTransaction(); + using var raw = _writer.BeginAuthoritativeFreshBulkInsertScope( + enabled: true, + CancellationToken.None)!; + using (var failedFile = _writer.BeginTransaction()) + { + var fileId = InsertNewFile("src/reference-line-duplicate-sequence.cs"); + var exception = Assert.Throws(() => + _writer.InsertReferencesForNewFilesInAtomicFileScope( + CreateReferences(fileId, 1, "duplicate_sequence"), + refreshMutualRecursionFlags: false, + CancellationToken.None)); + Assert.Contains("ID floor was negative (-1)", exception.Message, StringComparison.Ordinal); + } + raw.Complete(); + outerTransaction.Commit(); + } + finally + { + DbWriter.AuthoritativeFreshRawInsertExecutingForTesting = previousRawHook; + } + + Assert.Equal( + ["insert_files", "read_reference_line_id_floor"], + rawWork.Select(work => work.Operation).ToArray()); + Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM reference_lines")); + Assert.Equal( + 0L, + ScalarLong( + "SELECT COUNT(*) FROM files WHERE path = 'src/reference-line-duplicate-sequence.cs'")); + } + public void Dispose() { _db.Dispose(); @@ -992,7 +1116,7 @@ private static ReferenceRecord[] CreateReferences( }) .ToArray(); - private void PrimeSequencesForInt64Returning() + private void PrimeSequencesForInt64Ids() { Execute(""" INSERT INTO files ( @@ -1022,6 +1146,20 @@ private long ScalarLong(string sql) return Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture); } + private long ReferenceLineId(long fileId, int line, string context) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = """ + SELECT id + FROM reference_lines + WHERE file_id = @file_id AND line = @line AND context = @context + """; + command.Parameters.AddWithValue("@file_id", fileId); + command.Parameters.AddWithValue("@line", line); + command.Parameters.AddWithValue("@context", context); + return Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture); + } + private long ResourceListGeneration() => ScalarLong(""" SELECT CAST(value AS INTEGER) diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index dfaf1940e1..4ee6d11863 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -9492,7 +9492,7 @@ public void ReferenceLineLookup_BatchedInputUsesUniqueAutoIndexPlan() } [Fact] - public void ReferenceLineFreshInsert_ReturnsOnlyIdAndInputOrdinal() + public void ReferenceLineProviderFreshInsert_ReturnsOnlyIdAndInputOrdinal() { const int StatementRowCount = 4; var sql = DbWriter.BuildReferenceLineInsertSqlForTesting(StatementRowCount); @@ -9537,6 +9537,90 @@ public void ReferenceLineFreshInsert_ReturnsOnlyIdAndInputOrdinal() Assert.All(returned, static row => Assert.True(row.Id > 0)); } + [Fact] + public void AuthoritativeFreshReferenceLineInsert_AssignsContiguousIdsWithoutReturningScan() + { + const int statementRowCount = 4; + const long firstId = 50_000; + var sql = DbWriter.BuildAuthoritativeFreshReferenceLineInsertSqlForTesting(statementRowCount); + + Assert.Contains( + "WITH input(input_ordinal, file_id, line, context)", + sql, + StringComparison.Ordinal); + Assert.Contains("INSERT INTO reference_lines (id, file_id, line, context)", sql, StringComparison.Ordinal); + Assert.Contains("SELECT ?1 + input_ordinal", sql, StringComparison.Ordinal); + Assert.DoesNotContain("RETURNING", sql, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("SELECT input_ordinal", sql, StringComparison.Ordinal); + Assert.Contains("?13", sql, StringComparison.Ordinal); + Assert.DoesNotContain("?14", sql, StringComparison.Ordinal); + + using (var command = _db.Connection.CreateCommand()) + { + command.CommandText = "EXPLAIN QUERY PLAN " + + DbWriter.AuthoritativeFreshReferenceLineIdFloorSqlForTesting; + var plan = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + plan.Add(reader.GetString(3)); + Assert.Contains( + plan, + detail => detail.Contains("SEARCH reference_lines", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + plan, + detail => detail.Contains("SCAN reference_lines", StringComparison.OrdinalIgnoreCase)); + } + + var fileIds = Enumerable.Range(0, statementRowCount) + .Select(row => UpsertTestFile( + $"src/reference-line-raw-{row}.cs", + checksum: $"reference-line-raw-{row}")) + .ToArray(); + using (var command = _db.Connection.CreateCommand()) + { + command.CommandText = sql; + Bind(command, fileIds); + Assert.Equal(statementRowCount, command.ExecuteNonQuery()); + } + + using (var command = _db.Connection.CreateCommand()) + { + command.CommandText = "EXPLAIN QUERY PLAN " + sql; + Bind(command, fileIds); + var plan = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + plan.Add(reader.GetString(3)); + Assert.DoesNotContain( + plan, + detail => detail.Contains("CORRELATED", StringComparison.OrdinalIgnoreCase)); + } + + using (var command = _db.Connection.CreateCommand()) + { + command.CommandText = "SELECT id FROM reference_lines ORDER BY id"; + using var reader = command.ExecuteReader(); + var ids = new List(); + while (reader.Read()) + ids.Add(reader.GetInt64(0)); + Assert.Equal( + Enumerable.Range(0, statementRowCount).Select(offset => firstId + offset), + ids); + } + + static void Bind(SqliteCommand command, IReadOnlyList fileIds) + { + command.Parameters.AddWithValue("?1", firstId); + for (var row = 0; row < statementRowCount; row++) + { + var parameterBase = row * 3 + 1; + command.Parameters.AddWithValue($"?{parameterBase + 1}", fileIds[row]); + command.Parameters.AddWithValue($"?{parameterBase + 2}", row + 20); + command.Parameters.AddWithValue($"?{parameterBase + 3}", $"raw-文脈-{row}-😀"); + } + } + } + [Fact] public void BatchNumericParameterSql_IsOneOriginForTwentyFiveColumnSymbolsAndReferences() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 19b082d068..79c3803933 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -3559,11 +3559,11 @@ FROM symbol_references AS r } [Fact] - public void Run_FreshFullScan_RawReturningFailureRollsBackFileAndSerialConsumerContinues() + public void Run_FreshFullScan_RawChangedRowValidationFailureRollsBackFileAndSerialConsumerContinues() { var projectRoot = CreateTempProject(); var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); - var previousRowHook = DbWriter.AuthoritativeFreshRawReturningRowForTesting; + var previousChangedRowsHook = DbWriter.AuthoritativeFreshRawChangedRowCountForTesting; var previousRawHook = DbWriter.AuthoritativeFreshRawInsertExecutingForTesting; var previousRawScopeHook = DbWriter.AuthoritativeFreshRawInsertScopeDisposedForTesting; var rawWork = new List(); @@ -3580,16 +3580,16 @@ public void Run_FreshFullScan_RawReturningFailureRollsBackFileAndSerialConsumerC $"def caller_{index}():\n target_{index}()\n"); } - DbWriter.AuthoritativeFreshRawReturningRowForTesting = row => + DbWriter.AuthoritativeFreshRawChangedRowCountForTesting = change => { - var transformed = previousRowHook?.Invoke(row) ?? row; - if (transformed.Operation == "insert_reference_lines" + var actual = previousChangedRowsHook?.Invoke(change) ?? change.ActualChangedRows; + if (change.Operation == "insert_reference_lines" && Interlocked.Exchange(ref injected, 1) == 0) { throw new InvalidOperationException( - "injected raw RETURNING failure after the first row"); + "injected raw changed-row validation failure"); } - return transformed; + return actual; }; DbWriter.AuthoritativeFreshRawInsertExecutingForTesting = work => { @@ -3652,7 +3652,7 @@ public void Run_FreshFullScan_RawReturningFailureRollsBackFileAndSerialConsumerC } finally { - DbWriter.AuthoritativeFreshRawReturningRowForTesting = previousRowHook; + DbWriter.AuthoritativeFreshRawChangedRowCountForTesting = previousChangedRowsHook; DbWriter.AuthoritativeFreshRawInsertExecutingForTesting = previousRawHook; DbWriter.AuthoritativeFreshRawInsertScopeDisposedForTesting = previousRawScopeHook; DeleteDirectory(projectRoot); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerInitialFullIndexPerformanceTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerInitialFullIndexPerformanceTests.cs index 8e996dc02d..562d8752e9 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerInitialFullIndexPerformanceTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerInitialFullIndexPerformanceTests.cs @@ -122,7 +122,7 @@ private void RunInitialFullIndexMixedLanguageContract(int copiesPerLanguage, Tim }); Assert.All( rawWork.Where(work => work.Operation == "insert_reference_lines"), - work => Assert.Equal(work.StatementRows * 3, work.BoundParameterCount)); + work => Assert.Equal(work.StatementRows * 3 + 1, work.BoundParameterCount)); var rawScope = Assert.Single(rawScopeSnapshots); Assert.True(rawScope.Completed); Assert.Equal(32, rawScope.Capacity); From e746c23c495084242991ae8027391f71c50331f3 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 29 Aug 2026 23:47:33 +0900 Subject: [PATCH 07/11] Start scheduled tail extraction in the first worker wave --- DEVELOPER_GUIDE.md | 22 ++- TESTING_GUIDE.md | 4 +- .../+initial-full-index-tail-first.changed.md | 14 ++ ...mmandRunner.FullScan.ExtractionPipeline.cs | 11 ++ ...ommandRunner.FullScan.ExtractionWorkers.cs | 7 +- .../IndexCommandRunnerFullScanTests.cs | 143 ++++++++++++++++++ 6 files changed, 188 insertions(+), 13 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-tail-first.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 33187abb6f..90feb97636 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -336,10 +336,15 @@ body. To keep a large file near the input tail from starting only in the final worker wave, they probe at most the last `min(4 * workers, 64)` work items and claim known, indexable sizes largest-first. Equal sizes, unavailable metadata, and files already above the configured size cap retain their original order. -The target array and logical file indexes never move, serial hook/filter paths -do not probe, and the bounded completion queue still publishes in completion -order. Keep this tail probe and its schedule state independent of repository -size; an all-file metadata pass can regress network and virtual filesystems. +Workers consume that scheduled suffix before the unscheduled prefix, so its +largest eligible candidates enter the first worker wave; after the schedule is +exhausted, prefix ordinals resume in their original order. Together those two +segments form an exactly-once permutation, and only then does the existing +sparse logical-file mapping apply. The target array and logical file indexes +never move, serial hook/filter paths do not probe, and the bounded completion +queue still publishes in completion order. Keep this tail probe and its +schedule state independent of repository size; an all-file metadata pass can +regress network and virtual filesystems. Parallel full-scan workers also carry symbol-preparation state to the single persistence consumer. Reuse the worker's family-scope key and completed C# @@ -4589,9 +4594,12 @@ parallel full scan は extraction 本体を共有dynamic claimで配分します 最後のworker waveまで開始されないことを防ぐため、末尾の `min(4 * workers, 64)` work itemだけをprobeし、size取得済みかつ上限内のfileを大きい順に claimします。同一size、metadata取得不能、設定size上限を既に超えるfileは元順を維持します。 -target arrayと論理file indexは並べ替えず、serialなhook/filter経路はprobeせず、bounded completion -queueは引き続き完了順でpublishします。network/virtual filesystemで全file metadata passへ -退行しないよう、tail probeとschedule stateをrepository規模に依存しない固定上限に保ってください。 +workerはこのschedule済みsuffixを未scheduleのprefixより先に消費するため、末尾で最大の対象候補も +最初のworker waveへ入ります。schedule消費後はprefix ordinalを元順で再開し、両segment全体で +exactly-onceのpermutationを作ってから既存のsparseな論理file mappingを適用します。target arrayと +論理file indexは並べ替えず、serialなhook/filter経路はprobeせず、bounded completion queueは +引き続き完了順でpublishします。network/virtual filesystemで全file metadata passへ退行しないよう、 +tail probeとschedule stateをrepository規模に依存しない固定上限に保ってください。 parallel full scan の worker は、symbol preparation の状態も single persistence consumer へ 引き渡します。worker が解決した family-scope key と完了済みの C# source observation を再利用し、 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 7e5639cdc5..07e1d64d76 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -470,7 +470,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result pin the full-scan preparation handoff at its exceptional boundaries. Require source/scope call counts of 1/0 for a parallel C# symbol cap, 1/1 for a reference cap, and 0/0 for a generated-suppressed C# file. The hook fixture must force serial extraction, call each stage once, and keep all hook-mutated partial symbols on the rebuilt family key. Keep `FamilyScopeApplied` explicit: a carried `null` key may mean a legitimately applied null scope, while the symbol-cap fixture is explicitly unprepared. Preserve the separate Python symbol-cap fixture so this C# handoff coverage does not replace language-neutral cap behavior. - `IndexCommandRunnerTests.CalculateDefaultIndexParallelism_CapsAutomaticWorkersWithoutLoweringSmallHosts` uses a five-value processor-count matrix to keep automatic full-scan workers equal to the available CPUs through eight, cap larger hosts at eight, and leave the explicit worker maximum covered by the parser boundary tests. -- `IndexCommandRunnerTests.BuildFullScanExtractionTailSchedule_*` treats long-tail scheduling as a bounded-work contract rather than a wall-clock benchmark. With at most one worker, or when all work fits in the first worker wave (`workItemCount <= workerCount`), require an empty schedule and zero length probes. Otherwise keep probes limited to the final `min(4 * workers, 64)` work ordinals, sort known in-limit sizes descending with stable ties, retain unavailable and over-limit entries in source order, swallow only expected metadata failures, and propagate cancellation before another probe. The returned schedule must remain at most 64 integers and contain work ordinals rather than logical file indexes; pin the resolver directly so sparse `ExtractionFileIndexes` map those ordinals deterministically to file indexes while a missing mapping preserves ordinal identity. +- `IndexCommandRunnerTests.BuildFullScanExtractionTailSchedule_*` treats long-tail scheduling as a bounded-work contract rather than a wall-clock benchmark. With at most one worker, or when all work fits in the first worker wave (`workItemCount <= workerCount`), require an empty schedule and zero length probes. Otherwise keep probes limited to the final `min(4 * workers, 64)` work ordinals, sort known in-limit sizes descending with stable ties, retain unavailable and over-limit entries in source order, swallow only expected metadata failures, and propagate cancellation before another probe. The returned schedule must remain at most 64 integers and contain work ordinals rather than logical file indexes. `ResolveFullScanExtractionWorkOrdinal_*` pins schedule-first then original-prefix ordering, empty-schedule identity, exactly-once permutation, and composition with sparse `ExtractionFileIndexes`; `Run_InitialFullIndex_TailScheduleFeedsFirstParallelWorkerWave` uses two blocked workers and signals instead of sleeps to prove the scheduled suffix supplies the first wave. Keep logical file mapping after ordinal resolution, and preserve queue capacity, worker count, serial hook/filter routing, and completion-order publication. - `IndexCommandRunnerTests.SymbolExtractionWorker_LegacyEnvironmentHooksAreIgnored_Issue3398` launches the isolated symbol worker to prove legacy worker environment variables are ignored. Its callback budget includes process startup and is intentionally wider than ordinary in-process checks so local process load does not turn the legacy-env regression check into a timeout flake (#3863). - `IndexCommandRunnerTests.SymbolExtractionWorker_Utf8RequestsPreserveUnicodeAcrossLanguages` @@ -1626,7 +1626,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" full-scan preparation handoff の例外境界を固定します。parallel C# symbol cap はsource/scope call countを1/0、reference capは1/1、generated suppression対象のC# fileは0/0にしてください。hook fixtureはserial extractionを強制し、各stageを1回だけ呼び、hook mutation後の全partial symbolが再構築済みfamily keyと一致することを検証します。carried keyが`null`でも正当にnull scopeを適用済みの場合がある一方、symbol-cap fixtureは明示的に未準備なので、`FamilyScopeApplied`を独立して固定してください。このC# handoff coverageが言語共通cap挙動を置き換えないよう、別のPython symbol-cap fixtureも維持します。 - `IndexCommandRunnerTests.CalculateDefaultIndexParallelism_CapsAutomaticWorkersWithoutLoweringSmallHosts` processor count 5値の matrix で、8 CPU までは利用可能 CPU 数と automatic full-scan worker 数を一致させ、それより大きい host は8 workerに抑えます。明示指定の最大値は既存 parser boundary test が引き続き固定します。 -- `IndexCommandRunnerTests.BuildFullScanExtractionTailSchedule_*` はlong-tail schedulingをwall-clock benchmarkではなくbounded-work契約として固定します。workerが1以下、または全workが最初のworker waveに収まる場合（`workItemCount <= workerCount`）はscheduleを空にし、length probeを0回にしてください。それ以外ではprobeを末尾の`min(4 * workers, 64)` work ordinalだけに制限し、取得済みで上限内のsizeをstable tie付き降順にし、取得不能または上限超過entryはsource順を維持してください。想定内のmetadata failureだけを吸収し、次のprobe前にcancellationを伝播します。返すscheduleは最大64 integerのままにし、論理file indexではなくwork ordinalを保持してください。resolverを直接固定し、sparseな`ExtractionFileIndexes`がordinalを決定的にfile indexへmappingし、mapping未指定時はordinal identityを維持することを検証します。 +- `IndexCommandRunnerTests.BuildFullScanExtractionTailSchedule_*` はlong-tail schedulingをwall-clock benchmarkではなくbounded-work契約として固定します。workerが1以下、または全workが最初のworker waveに収まる場合（`workItemCount <= workerCount`）はscheduleを空にし、length probeを0回にしてください。それ以外ではprobeを末尾の`min(4 * workers, 64)` work ordinalだけに制限し、取得済みで上限内のsizeをstable tie付き降順にし、取得不能または上限超過entryはsource順を維持してください。想定内のmetadata failureだけを吸収し、次のprobe前にcancellationを伝播します。返すscheduleは最大64 integerのままにし、論理file indexではなくwork ordinalを保持してください。`ResolveFullScanExtractionWorkOrdinal_*` でschedule優先→元順prefix、空scheduleのidentity、exactly-once permutation、sparseな`ExtractionFileIndexes`との合成を固定し、`Run_InitialFullIndex_TailScheduleFeedsFirstParallelWorkerWave`では2 workerをblockしてsleepなしのsignal同期により最初のwaveをschedule済みsuffixが供給することを証明します。論理file mappingはordinal解決後に保ち、queue容量、worker数、serialなhook/filter経路、完了順publishを変えないでください。 - `IndexCommandRunnerTests.SymbolExtractionWorker_LegacyEnvironmentHooksAreIgnored_Issue3398` isolated symbol worker を起動し、legacy worker 環境変数が無視されることを検証します。この callback budget はプロセス起動時間も含むため、通常の in-process チェックより意図的に広く取り、ローカル負荷で legacy-env 回帰テストが timeout flake にならないようにします（#3863）。 - `IndexCommandRunnerTests.SymbolExtractionWorker_Utf8RequestsPreserveUnicodeAcrossLanguages` diff --git a/changelog.d/unreleased/+initial-full-index-tail-first.changed.md b/changelog.d/unreleased/+initial-full-index-tail-first.changed.md new file mode 100644 index 0000000000..48580f7530 --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-tail-first.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionWorkers.cs +--- + +## English + +- **Cold full scans start scheduled tail work in the first worker wave** — The bounded cross-language tail schedule is now consumed before the untouched prefix, so large eligible files near the scan tail begin promptly while every work item still runs exactly once. + +## 日本語 + +- **初回full scanの最初のworker waveでtail scheduleを開始** — 全言語共通のbounded tail scheduleを未変更のprefixより先に消費し、scan末尾付近の大きな対象fileを早く開始しながら、全work itemを引き続き厳密に1回ずつ処理します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs index b5c3ae072f..b124cd5bda 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs @@ -513,6 +513,17 @@ or ArgumentException return schedule; } + internal static int ResolveFullScanExtractionWorkOrdinal( + int extractionIndex, + int[] extractionTailSchedule) + // The schedule is a permutation of the contiguous input suffix. Consume it + // first, then map the remaining claims directly onto the unscheduled prefix; + // no repository-sized visited set or second ordering buffer is needed. + // scheduleは連続suffixのpermutationなので、先に消費した後は残りをprefixへ直結する。 + => extractionIndex < extractionTailSchedule.Length + ? extractionTailSchedule[extractionIndex] + : extractionIndex - extractionTailSchedule.Length; + internal static int ResolveFullScanExtractionFileIndex( IReadOnlyList? extractionFileIndexes, int workOrdinal) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionWorkers.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionWorkers.cs index 774b050402..5814ee0255 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionWorkers.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionWorkers.cs @@ -69,10 +69,9 @@ private static Task[] StartFullScanExtractionWorkers( if (extractionIndex >= extractionWorkItemCount) break; - var tailStart = extractionWorkItemCount - extractionTailSchedule.Length; - var workOrdinal = extractionIndex >= tailStart - ? extractionTailSchedule[extractionIndex - tailStart] - : extractionIndex; + var workOrdinal = ResolveFullScanExtractionWorkOrdinal( + extractionIndex, + extractionTailSchedule); var fileIndex = ResolveFullScanExtractionFileIndex( extractionFileIndexes, workOrdinal); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 79c3803933..93400b6138 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -1354,6 +1354,149 @@ public void BuildFullScanExtractionTailSchedule_PrioritizesLargestKnownFilesWith Assert.Equal([8, 4, 5, 7, 3, 9, 2, 6], schedule); } + [Fact] + public void ResolveFullScanExtractionWorkOrdinal_ConsumesTailPermutationThenPrefixExactlyOnce() + { + int[] schedule = [8, 4, 5, 7, 3, 9, 2, 6]; + + var resolved = Enumerable.Range(0, 10) + .Select(extractionIndex => + IndexCommandRunner.ResolveFullScanExtractionWorkOrdinal( + extractionIndex, + schedule)) + .ToArray(); + + Assert.Equal([8, 4, 5, 7, 3, 9, 2, 6, 0, 1], resolved); + Assert.Equal(Enumerable.Range(0, 10), resolved.Order()); + Assert.Equal(10, resolved.Distinct().Count()); + } + + [Fact] + public void ResolveFullScanExtractionWorkOrdinal_EmptySchedulePreservesOriginalOrder() + { + int[] schedule = []; + + var resolved = Enumerable.Range(0, 6) + .Select(extractionIndex => + IndexCommandRunner.ResolveFullScanExtractionWorkOrdinal( + extractionIndex, + schedule)) + .ToArray(); + + Assert.Equal(Enumerable.Range(0, 6), resolved); + } + + [Fact] + public void ResolveFullScanExtractionWorkOrdinal_ComposesWithSparseFileIndexes() + { + int[] schedule = [4, 3]; + int[] extractionFileIndexes = [11, 3, 17, 5, 23]; + + var fileIndexes = Enumerable.Range(0, extractionFileIndexes.Length) + .Select(extractionIndex => + IndexCommandRunner.ResolveFullScanExtractionFileIndex( + extractionFileIndexes, + IndexCommandRunner.ResolveFullScanExtractionWorkOrdinal( + extractionIndex, + schedule))) + .ToArray(); + + Assert.Equal([23, 5, 11, 3, 17], fileIndexes); + } + + [Fact] + public async Task Run_InitialFullIndex_TailScheduleFeedsFirstParallelWorkerWave() + { + var projectRoot = CreateTempProject(); + var previousContentLoadHook = + IndexCommandRunner.FullScanFileContentLoadForTesting; + using var firstWaveReady = new CountdownEvent(2); + using var releaseFirstWave = new ManualResetEventSlim(); + using var runFinished = new ManualResetEventSlim(); + var firstWavePaths = new ConcurrentQueue(); + Task<(int ExitCode, JsonElement Json)>? runTask = null; + var claimedCount = 0; + try + { + for (var index = 0; index < 10; index++) + { + File.WriteAllText( + Path.Combine(projectRoot, $"item-{index:D2}.json"), + $"{{\"value\":{index}}}\n"); + } + var scanTargets = new FileIndexer( + projectRoot, + ignoreCase: false, + ignoreRuleRoot: null) + .ScanFilesDetailedWithIndexingTargets() + .IndexingTargets; + Assert.Equal(10, scanTargets.Count); + var expectedSchedule = IndexCommandRunner.BuildFullScanExtractionTailSchedule( + scanTargets.Count, + workerCount: 2, + maxFileSizeBytes: long.MaxValue, + workOrdinal => new FileInfo(scanTargets[workOrdinal].FilePath).Length, + CancellationToken.None); + Assert.Equal([2, 3], expectedSchedule.Take(2)); + var expectedFirstWavePaths = expectedSchedule + .Take(2) + .Select(workOrdinal => scanTargets[workOrdinal].DisplayRelativePath) + .Order(StringComparer.Ordinal) + .ToArray(); + + IndexCommandRunner.FullScanFileContentLoadForTesting = + path => + { + previousContentLoadHook?.Invoke(path); + if (Interlocked.Increment(ref claimedCount) > 2) + return; + + firstWavePaths.Enqueue(path); + firstWaveReady.Signal(); + if (!releaseFirstWave.Wait(TimeSpan.FromSeconds(30))) + { + throw new TimeoutException( + "The first full-scan worker wave was not released."); + } + }; + runTask = Task.Run(() => + { + try + { + return RunAndCaptureJson( + [projectRoot, "--parallelism", "2", "--json", "--quiet"]); + } + finally + { + runFinished.Set(); + } + }); + + Assert.True( + firstWaveReady.Wait(TimeSpan.FromSeconds(30)), + "Both first-wave full-scan workers did not claim work."); + Assert.Equal( + expectedFirstWavePaths, + firstWavePaths.Order(StringComparer.Ordinal)); + + releaseFirstWave.Set(); + var (exitCode, json) = await runTask; + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + } + finally + { + releaseFirstWave.Set(); + var cleanupSafe = runTask == null + || runFinished.Wait(TimeSpan.FromSeconds(30)); + IndexCommandRunner.FullScanFileContentLoadForTesting = + previousContentLoadHook; + SqliteConnection.ClearAllPools(); + if (cleanupSafe) + DeleteDirectory(projectRoot); + } + } + [Theory] [InlineData(6, 1)] [InlineData(3, 4)] From a2cdbba65be86b3b85bf60f389ea1f5caf14291b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 30 Aug 2026 00:18:49 +0900 Subject: [PATCH 08/11] Avoid redundant EOF probes during cold reads --- DEVELOPER_GUIDE.md | 8 +- TESTING_GUIDE.md | 12 +- ...ial-full-index-content-snapshot.changed.md | 15 + .../Scanning/FileContentLoader.RawBytes.cs | 93 ++++-- .../FileContentLoader.UnknownLanguage.cs | 43 ++- .../Indexer/Scanning/FileContentLoader.cs | 23 +- .../FileIndexerContentLoadingTests.cs | 302 ++++++++++++++++++ tests/CodeIndex.Tests/FileIndexerTests.cs | 7 + 8 files changed, 450 insertions(+), 53 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-content-snapshot.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 90feb97636..024c3e1150 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2776,8 +2776,8 @@ Process exit codes are coarse (`0` success including valid zero-row queries, `1` - **Reference contexts materialize only for emitted rows** — Built-in core, functional-language, and Solidity extractors pass the physical source-line instance through their emitters and trim it only after at least one reference survives filtering and deduplication. The deferred normalizer must key the source line by the emitted physical line number and require reference identity with that raw line; value equality would rewrite derived XAML, Razor, plugin, or other specialized contexts. Columns remain based on the untrimmed physical line, and stateful emitters must still advance on reference-free lines. - **Language capability patterns remain typed at the integration boundary** — CLI/MCP `languages` rows expose suffix-only `extensions`, literal `exact_filenames`, and ``-rendered `filename_prefix_patterns`. `legacy_patterns` preserves the former combined list during deprecation, and `pattern_provenance` identifies built-in, plugin/pattern, and language-map override ownership. Round-trip tests feed every advertised typed pattern back through `FileIndexer.DetectLanguage` (#4617). - **Ambiguous source extensions stay explicit** — `.m` and `.pl` are not assigned to Objective-C and Perl by default. After language-map overrides and built-in exact/prefix filename rules, `FileIndexer` checks an authoritative recognized shebang whose first physical line is bounded to 256 bytes, then a 64 KiB bounded prefix for strong mutually exclusive Objective-C/MATLAB or Perl/Prolog markers, then at most 256 entries per ancestor directory for conservative project markers. A first line that reaches the shebang boundary without a terminator falls through instead of selecting an interpreter. Conflicting or weak evidence is indexed as `ambiguous_m` / `ambiguous_pl`; unresolved `.m` files run the bounded MATLAB and Objective-C symbol/reference paths after a shared position-preserving comment mask, while Prolog and `ambiguous_pl` advertise conservative reference/graph support and the ambiguous `.pl` bucket uses union symbol/reference rules without changing content-based classification. The detector owns the ordered candidate descriptors, filename patterns, exact content patterns, project markers, bounded shebang rules, and reason/confidence vocabulary; CLI/MCP `extension_lookup` diagnostics and dry-run `language_detections` consume that same source so catalog guidance cannot drift from indexing decisions (#4612, #4738, #4746, #4901). -- **Content reads aggregate open-handle metadata** — Authoritative raw loads, raw-chunk probes, the specialized C# prepass, and final unknown-language probes capture one initial and one final `FileHandleSnapshot` for every stable open. Each snapshot obtains length, mtime, and file identity together through one `GetFileInformationByHandle` call on Windows, one `fstat` call on macOS, or one fixed-layout `statx(..., AT_EMPTY_PATH, ...)` call on Linux; older or unsupported runtimes retain the managed multi-call fallback. The initial snapshot supplies both open-binding identity and the read baseline, while the final snapshot supplies both mutation metadata and the opened identity used against a separate current-path identity probe. Stable reads therefore use exactly two logical snapshots, and one bounded retry uses four, without changing the distinct raw-load, positive chunk-match, C# prepass, or unknown-language retry contracts. -- **Unknown-language membership uses one bounded file snapshot** — Exact filenames, registered extensions, pattern/plugin mappings, and the `.m` / `.pl` ambiguous detectors keep their existing precedence and I/O. Only the final extensionless or unregistered-extension fallback defers its script-header check: it fills at most 256 bytes with short-read-safe reads from one authorized handle and returns immediately for a recognized shebang or `#compdef`. Otherwise the same handle continues through a pooled fixed-size buffer to EOF or `max-file-bytes + 1`, retaining only the first 4096 bytes for UTF-16 BOM/parity detection while checking the whole stream for NUL and the strict sub-1024-byte Git LFS pointer shape. Length, mtime, and path identity changes discard the first snapshot and re-resolve, reauthorize, and reopen once. Full CLI scans, scoped dry-run probes, freshness scans, and MCP full/dry indexing all consume this `FileIndexer` boundary, so unknown-language diagnostics do not pay a separate full-file allocation or second stable open. +- **Content reads aggregate open-handle metadata** — Authoritative raw loads, raw-chunk probes, the specialized C# prepass, and final unknown-language probes capture one initial and one final `FileHandleSnapshot` for every stable open. Each snapshot obtains length, mtime, and file identity together through one `GetFileInformationByHandle` call on Windows, one `fstat` call on macOS, or one fixed-layout `statx(..., AT_EMPTY_PATH, ...)` call on Linux; older or unsupported runtimes retain the managed multi-call fallback. The initial snapshot supplies both open-binding identity and the read baseline, while the final snapshot supplies both mutation metadata and the opened identity used against a separate current-path identity probe. On attempt zero, full-content and negative scans stop at the initial length without an extra EOF or `ReadByte` growth probe; stability requires the actual byte count, final length/mtime/handle identity, and current-path identity to agree with that baseline. A changed snapshot reopens once and only the retry scans to bounded EOF, while a final handle length over the max-file cap still fails immediately as growth during the current read. Positive raw-chunk matches remain conservatively true. Stable reads therefore use exactly two logical snapshots, and one bounded retry uses four, without changing the distinct raw-load, positive chunk-match, C# prepass, or unknown-language retry contracts. +- **Unknown-language membership uses one bounded file snapshot** — Exact filenames, registered extensions, pattern/plugin mappings, and the `.m` / `.pl` ambiguous detectors keep their existing precedence and I/O. Only the final extensionless or unregistered-extension fallback defers its script-header check: it fills at most 256 bytes with short-read-safe reads from one authorized handle and returns immediately for a recognized shebang or `#compdef`. Otherwise the first attempt continues through a pooled fixed-size buffer only to the initial handle length, retaining the first 4096 bytes for UTF-16 BOM/parity detection while checking every consumed byte for NUL and the strict sub-1024-byte Git LFS pointer shape. Matching final length, actual byte count, mtime, and identity make that snapshot authoritative without an EOF probe; a change discards it and re-resolves, reauthorizes, and reopens once, and only that bounded retry continues to EOF or `max-file-bytes + 1`. Full CLI scans, scoped dry-run probes, freshness scans, and MCP full/dry indexing all consume this `FileIndexer` boundary, so unknown-language diagnostics do not pay a separate full-file allocation or second stable open. - **Dynamic reference-graph readiness follows extractor contracts** — when indexed Crystal, Groovy, Tcl, Prolog, or `ambiguous_pl` rows have a missing or stale symbol-extractor version stamp, status reports `dynamic_reference_graph_contract_stale` and keeps `reference_graph_complete` / `graph_data_current` false until a normal index refresh rewrites those rows (#4746). - **Hotspot marker fingerprints share one bounded tree traversal** — full/update CLI and MCP indexing compute C#, VB, F#, and MSBuild marker fingerprints together instead of walking the directory tree once per language. Each distinct marker glob retains the platform filesystem's matching behavior and is enumerated once per visited directory, while child directories are enumerated once; marker sets, budgets, truncation sentinels, and warning order remain isolated per language. The single-language API delegates to the same engine, preserving ignore rules, nested-repository/submodule boundaries, and MCP authorized-read failures. - **Lock-file dependency graphs model package relationships** — `packages.lock.json`, `package-lock.json`, and `npm-shrinkwrap.json` keep package declarations as symbols, but emit `dependency` references only for explicit parent-package to child-package entries. NuGet lock symbols and references preserve the current file, target/RID, parent package, and exact JSON property span; candidate resolution stays file-local, while file-level `deps` suppresses cross-file package-name inference. Normal index updates invalidate the prior dependency-lock extraction and reference-identity contracts, so `callers` identifies the requiring package without connecting unrelated lock files or collapsing repeated declarations to the first matching line (#4409, #4845). @@ -6966,8 +6966,8 @@ USER_GUIDEの[終了コード](USER_GUIDE.md#終了コード)セクションを - **reference context はrowを発行した行だけmaterialize** — built-inのcore、functional language、Solidity extractorは物理source lineのinstanceをemitterへ渡し、filterとdedupを通過したreferenceが1件以上ある場合だけ後段でtrimします。遅延normalizerは発行された物理line numberからsource lineを引き、そのraw lineとの参照同一性を必須にします。値一致にするとXAML、Razor、pluginなどの派生contextまで書き換えるためです。columnはtrim前の物理行を基準に保ち、stateful emitterはreferenceがない行でもstate更新を続けます。 - **integration boundary では language capability pattern の型を維持** — CLI/MCP の `languages` 行は suffix のみの `extensions`、literal な `exact_filenames`、`` 表記の `filename_prefix_patterns` を公開します。`legacy_patterns` は deprecation 中に従来の combined list を保持し、`pattern_provenance` は built-in、plugin/pattern、language-map override の所有元を示します。round-trip test は広告した全 typed pattern を `FileIndexer.DetectLanguage` に戻して検証します（#4617）。 - **曖昧な source extension は曖昧なまま明示** — `.m` と `.pl` を既定で Objective-C / Perl に割り当てません。language-map override と built-in の完全一致/prefix filename rule の後で、`FileIndexer` は先頭物理行を 256 byte に制限した authoritative な認識済み shebang、64 KiB 上限 prefix 内の相互排他的で強い Objective-C/MATLAB または Perl/Prolog marker、各 ancestor directory 最大 256 entry の保守的な project marker の順に確認します。行終端なしで shebang 境界に達した先頭行は interpreter を選択せず、後続判定へ進みます。競合または弱い証拠は `ambiguous_m` / `ambiguous_pl` として index し、未確定の `.m` は位置を保つ共通コメントマスクの後で上限付きの MATLAB / Objective-C symbol・reference 経路を実行します。一方、Prolog と `ambiguous_pl` は保守的な reference / graph 対応を広告し、曖昧な `.pl` bucket は content-based classification を変えずに symbol / reference rule の和集合を使います。順序付き candidate descriptor、filename pattern、正確な content pattern、project marker、上限付き shebang rule、reason/confidence 語彙は detector 自身が所有し、CLI/MCP の `extension_lookup` diagnostic と dry-run の `language_detections` は同じ source を使うため、catalog guidance と indexing 判定が乖離しません（#4612、#4738、#4746、#4901）。 -- **content read はopen済みhandleのmetadataを集約する** — authoritative raw load、raw-chunk probe、C#専用prepass、最終unknown-language probeは、stableなopenごとに最初と最後の`FileHandleSnapshot`を1つずつ取得します。各snapshotはWindowsの`GetFileInformationByHandle` 1回、macOSの`fstat` 1回、Linuxの固定layout `statx(..., AT_EMPTY_PATH, ...)` 1回によりlength・mtime・file identityをまとめて取得し、古いruntimeまたは未対応platformでは従来のmanaged multi-call fallbackを維持します。最初のsnapshotはopen bindingのidentityとread baselineを兼ね、最後のsnapshotはmutation metadataと、別途取得するcurrent-path identityとの比較に使うopened identityを兼ねます。これによりstable readは論理snapshotを厳密に2回、上限付きretry 1回では4回に保ちつつ、raw load、positive chunk match、C# prepass、unknown-languageで異なるretry契約を変更しません。 -- **未知言語の membership は1つの上限付き file snapshotを使う** — 完全一致 filename、登録済み extension、pattern/plugin mapping、`.m` / `.pl` の曖昧判定は従来の優先順位と I/O を維持します。最後の拡張子なし・未登録拡張子 fallback だけ script header 判定を遅延し、同じ認可済み handle から short read に耐える loop で最大256 byteを満たします。認識済み shebang / `#compdef` はそこで即時返却し、それ以外は同じ handle を pooled fixed-size buffer で EOF または `max-file-bytes + 1` まで読み進めます。UTF-16 BOM/parity 判定には先頭4096 byteだけを保持しつつ、全streamのNULと1024 byte未満の厳密なGit LFS pointer形を確認します。length、mtime、path identityが変化した最初のsnapshotは破棄し、resolve・authorize・openを1回だけやり直します。CLI full scan、scoped dry-run、freshness scan、MCP full/dry indexingはいずれもこの`FileIndexer`境界を共有するため、未知言語diagnosticのための別のfull-file allocationやstable fileの2回目openは発生しません。 +- **content read はopen済みhandleのmetadataを集約する** — authoritative raw load、raw-chunk probe、C#専用prepass、最終unknown-language probeは、stableなopenごとに最初と最後の`FileHandleSnapshot`を1つずつ取得します。各snapshotはWindowsの`GetFileInformationByHandle` 1回、macOSの`fstat` 1回、Linuxの固定layout `statx(..., AT_EMPTY_PATH, ...)` 1回によりlength・mtime・file identityをまとめて取得し、古いruntimeまたは未対応platformでは従来のmanaged multi-call fallbackを維持します。最初のsnapshotはopen bindingのidentityとread baselineを兼ね、最後のsnapshotはmutation metadataと、別途取得するcurrent-path identityとの比較に使うopened identityを兼ねます。attempt 0のfull-content readとnegative scanはinitial lengthで停止し、余分なEOFまたは`ReadByte` growth probeを行いません。実読byte数、final length/mtime/handle identity、current-path identityがbaselineと一致した場合だけstableと判定します。変化時は1回だけ再openし、bounded EOF scanはretryだけが行います。一方、final handle lengthがmax-file上限を超えた場合は、従来どおりそのread中のgrowthとして即時失敗します。positive raw-chunk matchは保守的なtrueを維持します。これによりstable readは論理snapshotを厳密に2回、上限付きretry 1回では4回に保ちつつ、raw load、positive chunk match、C# prepass、unknown-languageで異なるretry契約を変更しません。 +- **未知言語の membership は1つの上限付き file snapshotを使う** — 完全一致 filename、登録済み extension、pattern/plugin mapping、`.m` / `.pl` の曖昧判定は従来の優先順位と I/O を維持します。最後の拡張子なし・未登録拡張子 fallback だけ script header 判定を遅延し、同じ認可済み handle から short read に耐える loop で最大256 byteを満たします。認識済み shebang / `#compdef` はそこで即時返却し、それ以外の初回attemptは同じhandleをpooled fixed-size bufferでinitial handle lengthまでだけ読み進めます。UTF-16 BOM/parity判定用の先頭4096 byteを保持し、その範囲を含む読取済み全byteでNULと1024 byte未満の厳密なGit LFS pointer形を確認します。final length、実読byte数、mtime、identityが一致すればEOF probeなしでauthoritativeとし、変化時はsnapshotを破棄してresolve・authorize・openを1回だけやり直します。EOFまたは`max-file-bytes + 1`まで進むのはこのbounded retryだけです。CLI full scan、scoped dry-run、freshness scan、MCP full/dry indexingはいずれもこの`FileIndexer`境界を共有するため、未知言語diagnosticのための別のfull-file allocationやstable fileの2回目openは発生しません。 - **動的言語の reference-graph readiness は extractor contract に従う** — index 済みの Crystal、Groovy、Tcl、Prolog、`ambiguous_pl` row で symbol-extractor version stamp が欠落または古い場合、status は `dynamic_reference_graph_contract_stale` を報告し、通常の index refresh が対象 row を更新するまで `reference_graph_complete` / `graph_data_current` を false に保ちます（#4746）。 - **hotspot marker fingerprint は上限付きtree traversalを1回共有** — full/update CLIとMCP indexingは、directory treeを言語ごとに歩かず、C#、VB、F#、MSBuildのmarker fingerprintをまとめて計算します。各directoryでは固有marker globごとにplatform filesystemのmatching挙動を保って1回ずつ列挙し、child directoryも1回だけ列挙する一方、marker集合、budget、truncation sentinel、warning順は言語別に分離します。single-language APIも同じengineへ委譲し、ignore rule、nested repository/submodule境界、MCP authorized read failureを維持します。 - **lock file の依存グラフは package 間の関係をモデル化** — `packages.lock.json`、`package-lock.json`、`npm-shrinkwrap.json` は package 宣言を symbol として保持しますが、`dependency` reference は明示された親 package → 子 package の項目だけに出力します。NuGet lock の symbol / reference は現在の file、target/RID、親 package、正確な JSON property span を保持し、candidate 解決を file 内に限定します。file 単位の `deps` は package 名による file 間推論を抑止し、通常の index update は以前の dependency-lock 抽出 contract と reference-identity contract を無効化します。そのため、`callers` は無関係な lock file を接続したり、反復宣言を最初の一致行へ畳み込んだりせず、要求元 package を特定できます（#4409、#4845）。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 07e1d64d76..342420b075 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -194,8 +194,8 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result TypeScript same-line sibling-class coverage keeps distinct and identical method-name cases in one source, using four class containers to preserve attribution diagnostics. - `FileIndexerTests.cs`, `FileIndexerContentLoadingTests.cs`, `FileIndexerTestSupport.cs`, `NormalizedContentFactsTests.cs`, `NormalizedContentFactsLoadingTests.cs` File scanning, language detection, scan-result language reuse, content-sensitive header safeguards, content loading/canonicalization, checksum, Git LFS pointer detection, and record-building behavior, including authoritative language-map suffix overrides across exact/prefix/extension rules, fail-visible child-map probe/read diagnostics that block parent inheritance, typed `languages` pattern schemas with provenance and detection round trips, extensionless/unknown first-line `#compdef` recognition, extensionless/unknown/ambiguous-extension shebang detection's override precedence, `detection_source`, 256-byte first-line cap, binary/NUL-byte rejection, conservative `.m` / `.pl` content/project classification with explicit ambiguity buckets, and Windows-only >=260-character path walker/purge coverage. Keep `.h` lexical-detection coverage consolidated across comment-only markers, spliced strings/comments and delimiters, long macro/raw-string/block-comment state, UTF-8 byte-budget sampling, partial-token sample boundaries, long physical lines, genuine mixed C/C++ code, and source/confidence metadata. Shared `FileIndexerTests` helpers live in `FileIndexerTestSupport.cs`. - Open-handle metadata coverage must compare the native aggregate snapshot with managed handle length, mtime, and the existing file identity on every supported OS. Keep stable authoritative loads, negative and positive raw-chunk probes, C# raw-negative/raw-positive prepasses, recognized unknown headers, and full unknown coverage at exactly two logical handle snapshots; include a stable identity-bound load to prove the initial snapshot is reused for binding. Same-size/same-mtime atomic replacement must reopen once and capture exactly four snapshots, while a retarget rejected during binding captures only the initial snapshot. Do not replace these deterministic call counts with timing assertions. - Unknown-language snapshot coverage must keep 1-3-byte short reads, the 256-byte recognized-header boundary, a NUL before versus after that boundary, UTF-16 LE/BE BOM and parity samples crossing chunk boundaries, UTF-32 rejection, strict Git LFS `<1024` / `==1024` behavior, late NUL offsets, exact/growing max-file limits, cancellation, same-size/same-mtime atomic replacement, internal-symlink retargeting, bounded two-attempt retry, and 4 MiB allocation independence together. Stable final-unknown probes open once; a detected mutation may re-resolve, reauthorize, and reopen only once. Pair the `FileIndexerTests` contract with scoped `IndexCommandRunnerDryRunTests`, full-scan unknown-extension metadata, and MCP authorized full/dry scan coverage so no consumer restores a header-open plus payload-open sequence. + Open-handle metadata coverage must compare the native aggregate snapshot with managed handle length, mtime, and the existing file identity on every supported OS. Keep stable authoritative loads, negative and positive raw-chunk probes, C# raw-negative/raw-positive prepasses, recognized unknown headers, and full unknown coverage at exactly two logical handle snapshots; include a stable identity-bound load to prove the initial snapshot is reused for binding. Stable full and negative paths must record zero EOF-growth reads and zero `ReadByte` probes on attempt zero. Same-mtime growth, shrink then regrow, a final length over the cap, second-attempt growth, and same-size/same-mtime atomic replacement must pin one-open/two-snapshot versus two-open/four-snapshot behavior; a retarget rejected during binding captures only the initial snapshot. Do not replace these deterministic call counts with timing assertions. + Unknown-language snapshot coverage must keep 1-3-byte short reads, the 256-byte recognized-header boundary, a NUL before versus after that boundary, UTF-16 LE/BE BOM and parity samples crossing chunk boundaries, UTF-32 rejection, strict Git LFS `<1024` / `==1024` behavior, late NUL offsets, exact/growing max-file limits, cancellation, same-size/same-mtime atomic replacement, internal-symlink retargeting, bounded two-attempt retry, and 4 MiB allocation independence together. Stable final-unknown probes open once and stop at the initial length without an EOF growth read; a detected mutation may re-resolve, reauthorize, and reopen only once, and that retry retains the conventional bounded EOF scan. Pair the `FileIndexerTests` contract with scoped `IndexCommandRunnerDryRunTests`, full-scan unknown-extension metadata, and MCP authorized full/dry scan coverage so no consumer restores a header-open plus payload-open sequence. Ambiguous-extension diagnostic coverage keeps `.M` case normalization, ordered Objective-C/MATLAB candidates, aliases, exact/prefix filename precedence, the 256-byte unterminated-shebang boundary, shebang/content/project evidence, override examples, empty/binary outcomes, and detector reason/confidence parity together across `FileIndexerTests`, CLI `languages`, MCP `languages`, and index dry-run fixtures (#4901). Directory-entry snapshot coverage requires one ordered, scan-local filesystem enumeration for each normally visited root/child directory and shares it between the default case probe and entry processing. Keep custom case-probe invocation counts, the legacy injected `enumerateFiles` path, dangling-entry diagnostics, and single-error enumeration failures explicit when changing traversal. Project-marker fingerprint coverage requires the C#/VB/F#/MSBuild batch to share one directory-tree traversal while retaining one enumeration per distinct marker glob and one child-directory enumeration per visited directory. Pin known hashes and single-language delegation, independent budgets/truncation/warning order, ignore/nested-repository/submodule and platform-casing boundaries, traversal failure and cancellation propagation, and MCP fingerprint/authorization parity. Family-scope snapshot coverage must keep exact marker counts distinct until publication, then resolve only marker directories and their ancestors through the scan-cached per-directory case policy; use injected entry snapshots to cover case-sensitive children below insensitive roots and insensitive-child aliases without any post-scan filesystem probe or live marker enumeration. @@ -481,7 +481,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result exercises the production stream-response overload with a Japanese C# symbol and verifies one BOM-less, newline-terminated UTF-8 JSON frame. Keep the `StringWriter` protocol tests as the in-process diagnostic path while this test protects process stdout framing. - `BoundedLineReaderTests.ReadUtf8LineAsync_BuffersFramesWithoutDecoding`, `ReadUtf8LineAsync_EnforcesByteLimitBeforeGrowth`, and `ReadUtf8LineAsync_HandlesCrLfAcrossBufferBoundary` cover multi-frame remainder reuse, CRLF split across a 4 KiB read boundary, Unicode bytes, an unterminated final frame, stable EOF, and rejection at one byte over the negotiated cap. They protect direct worker-response deserialization without constructing decoded JSON strings. -- `IndexCommandRunnerTests`, `FileIndexerTests`, and `PerformanceTests` also cover `CSharpStaticInterfacePrepass` text, raw-byte, chunked raw-token, and streaming file contract probes. Stable candidate reads must authorize and open each file once, keep raw-negative reads to one bounded pass, and rewind that same handle only for a raw-positive full decode. A detected in-place or atomic-replacement mutation must discard that snapshot and reauthorize/reopen once so the prepass cannot diverge from the main indexing pass. Preserve UTF-8 / UTF-16, NUL rejection, growth, cancellation, and lexical-boundary behavior. The 576 KiB semantic-negative/positive allocation guard runs each probe 12 times and stays below 4 KiB of current-thread allocation so a whole-content mask cannot return. +- `IndexCommandRunnerTests`, `FileIndexerTests`, and `PerformanceTests` also cover `CSharpStaticInterfacePrepass` text, raw-byte, chunked raw-token, and streaming file contract probes. Stable candidate reads must authorize and open each file once, keep raw-negative reads to one initial-length-bounded pass with no EOF growth read, and rewind that same handle only for a raw-positive full decode; that decode also stops at the initial length without a `ReadByte` probe on attempt zero. A detected in-place or atomic-replacement mutation must discard that snapshot and reauthorize/reopen once so the prepass cannot diverge from the main indexing pass, and only the retry scans to bounded EOF. Preserve UTF-8 / UTF-16, NUL rejection, growth, cancellation, and lexical-boundary behavior. The 576 KiB semantic-negative/positive allocation guard runs each probe 12 times and stays below 4 KiB of current-thread allocation so a whole-content mask cannot return. - The parallel C# static-interface full-scan fixture uses 64 implementation files and treats one workspace lookup build as a performance contract. Keep the contract lookup attached to the immutable prepass snapshot across CLI full scan, scoped update, and MCP indexing; do not rebuild it once per C# file. - `Run_FullScan_PostPrepassCsharpContractLeavesReadinessPartialUntilCleanRetry` keeps the full-scan extraction state monotonic across fresh, rebuild, and incremental-existing routes. Its ordered Python-before-C# fixture must prove that the earlier raw chunk persists and remains searchable through both standard and trigram FTS after the later C# workspace snapshot drifts, with exactly one bulk-load optimization. - `Run_FullScan_FatalParallelResultKeepsWorkerResourcesAliveUntilPeersStop` blocks one C# symbols worker while a peer reports a fatal extraction stall. Keep the command return prompt, assert that neither worker completion nor artifact-cache clearing occurs before the blocked peer is released, and wait for both cleanup signals before restoring process-wide hooks or deleting the fixture. @@ -1353,8 +1353,8 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" TypeScript same-line sibling-class coverage は、distinct/identical method-name case を1 source にまとめ、4つの class container で attribution の診断性を維持します。 - `FileIndexerTests.cs`、`FileIndexerContentLoadingTests.cs`、`FileIndexerTestSupport.cs`、`NormalizedContentFactsTests.cs`、`NormalizedContentFactsLoadingTests.cs` ファイル走査、言語判定、scan result 言語の再利用、content loading / canonicalization、checksum、レコード構築のテスト。完全一致/prefix/extension rule を横断する authoritative な language-map suffix override、親継承を遮断して失敗を可視化する child map の probe/read diagnostic、provenance と detection round trip を持つ typed `languages` pattern schema、拡張子なし/未知拡張子の先頭行 `#compdef` 認識、拡張子なし/未知拡張子/曖昧拡張子の shebang 判定における override precedence、`detection_source`、「先頭物理行 256 byte 上限」、binary/NUL byte 除外、明示 ambiguity bucket を持つ保守的な `.m` / `.pl` content/project 分類、Windows 専用の 260 文字以上 path walker/purge カバレッジも含みます。`.h` の字句判定カバレッジは、コメントだけのマーカー、splice された文字列・コメント・delimiter、長いマクロ・raw string・block comment の状態、UTF-8 byte budget sampling、token 途中の sample 境界、長い物理行、実際の C/C++ 混在コード、判定元・信頼度 metadata を一まとまりで検証します。共有 `FileIndexerTests` helper は `FileIndexerTestSupport.cs` に置きます。 - open済みhandleのmetadata coverageでは、対応する全OSでnative aggregate snapshotをmanaged handleのlength・mtimeおよび既存file identityと比較します。stableなauthoritative load、negative/positive raw-chunk probe、C#のraw-negative/raw-positive prepass、認識済みunknown header、unknownのfull coverageを論理handle snapshot厳密2回に固定し、最初のsnapshotをbindingにも再利用することをstableなidentity-bound loadで証明してください。同一size/mtimeのatomic replacementは1回だけ再openしてsnapshot厳密4回、binding中に拒否するretargetは最初のsnapshotだけを取得します。これらの決定的call countをtiming assertionで置き換えないでください。 - 未知言語snapshotのcoverageでは、1〜3 byteのshort read、認識済みheaderの256 byte境界、その境界より前後のNUL、chunk境界をまたぐUTF-16 LE/BEのBOM/parity sample、UTF-32除外、厳密なGit LFS `<1024` / `==1024`、後半NUL offset、max-file上限一致とgrowth、cancellation、同一size/mtimeのatomic replacement、internal symlink retarget、最大2 attemptのretry、4 MiB payloadに比例しないallocationを一まとまりで維持してください。stableな最終unknown probeは1回だけopenし、mutation検出時もresolve・authorize・openの再試行は1回だけです。`FileIndexerTests`の契約をscoped `IndexCommandRunnerDryRunTests`、full-scanのunknown-extension metadata、MCPのauthorized full/dry scan coverageと対にし、どのconsumerもheader-openとpayload-openの2段階へ戻らないようにしてください。 + open済みhandleのmetadata coverageでは、対応する全OSでnative aggregate snapshotをmanaged handleのlength・mtimeおよび既存file identityと比較します。stableなauthoritative load、negative/positive raw-chunk probe、C#のraw-negative/raw-positive prepass、認識済みunknown header、unknownのfull coverageを論理handle snapshot厳密2回に固定し、最初のsnapshotをbindingにも再利用することをstableなidentity-bound loadで証明してください。stableなfull/negative経路ではattempt 0のEOF-growth readと`ReadByte` probeを0回に固定します。同一mtimeのgrowth、shrink後のregrow、final lengthの上限超過、2回目attempt中のgrowth、同一size/mtimeのatomic replacementで、open 1回/snapshot 2回とopen 2回/snapshot 4回の契約を固定してください。binding中に拒否するretargetは最初のsnapshotだけを取得します。これらの決定的call countをtiming assertionで置き換えないでください。 + 未知言語snapshotのcoverageでは、1〜3 byteのshort read、認識済みheaderの256 byte境界、その境界より前後のNUL、chunk境界をまたぐUTF-16 LE/BEのBOM/parity sample、UTF-32除外、厳密なGit LFS `<1024` / `==1024`、後半NUL offset、max-file上限一致とgrowth、cancellation、同一size/mtimeのatomic replacement、internal symlink retarget、最大2 attemptのretry、4 MiB payloadに比例しないallocationを一まとまりで維持してください。stableな最終unknown probeは1回だけopenしてEOF growth readなしでinitial lengthに停止します。mutation検出時もresolve・authorize・openの再試行は1回だけとし、そのretryだけが従来のbounded EOF scanを維持します。`FileIndexerTests`の契約をscoped `IndexCommandRunnerDryRunTests`、full-scanのunknown-extension metadata、MCPのauthorized full/dry scan coverageと対にし、どのconsumerもheader-openとpayload-openの2段階へ戻らないようにしてください。 曖昧拡張子 diagnostic の coverage は、`.M` の大小文字正規化、順序付き Objective-C/MATLAB candidate、alias、完全一致/prefix filename の優先順位、行終端なしで到達する 256 byte shebang 境界、shebang/content/project evidence、override 例、空・binary 入力の結果、detector の reason/confidence parity を `FileIndexerTests`、CLI `languages`、MCP `languages`、index dry-run fixture の間でまとめて維持します（#4901）。 directory entry snapshot の coverage では、通常scanで訪問するroot/child directoryごとに順序付き・scan-localなfilesystem列挙を1回だけ行い、既定case probeとentry処理で共有することを必須とします。traversal変更時はcustom case probeの呼出回数、旧injected `enumerateFiles`経路、dangling entry診断、列挙失敗が1 errorだけになる契約を明示的に維持してください。 project marker fingerprint の coverage では、C#/VB/F#/MSBuild batch が1回の directory-tree traversal を共有しつつ、各directoryで固有marker globごとに1回、child directoryは1回だけ列挙することを必須とします。既知hashとsingle-language delegation、言語別budget/truncation/warning順、ignore/nested repository/submoduleとplatform casingの境界、traversal failure/cancellationの伝播、MCP fingerprint/authorization parityを固定してください。family-scope snapshot の coverage は、publish まで marker count を完全一致で分離し、publish 後は marker directory とその祖先だけを scan 済み directory ごとの case policy で解決する契約を維持します。case-insensitive root 配下の case-sensitive child と case-insensitive child alias は injected entry snapshot で検証し、scan 後の filesystem probe や live marker 列挙が0であることも固定してください。 @@ -1637,7 +1637,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" 日本語の C# symbol で本番用 stream-response overload を実行し、BOM なし・改行終端の UTF-8 JSON frame が1件出ることを検証します。`StringWriter` の protocol tests は in-process diagnostic 経路として維持し、このテストで process stdout framing を固定します。 - `BoundedLineReaderTests.ReadUtf8LineAsync_BuffersFramesWithoutDecoding`、`ReadUtf8LineAsync_EnforcesByteLimitBeforeGrowth`、`ReadUtf8LineAsync_HandlesCrLfAcrossBufferBoundary` multi-frame remainder の再利用、4 KiB read 境界をまたぐ CRLF、Unicode byte、改行なし最終 frame、安定した EOF、合意済み上限を1 byte 超えた時点での拒否を検証します。decode 済み JSON string を作らず worker response を直接 deserialize する経路を固定します。 -- `IndexCommandRunnerTests`、`FileIndexerTests`、`PerformanceTests` は `CSharpStaticInterfacePrepass` のテキスト判定、raw-byte、chunked raw-token、streaming file 契約 probe も扱います。安定した候補読み取りは各 file を1回だけ認可・openし、raw-negative は bounded pass 1回に留め、raw-positive の full decode だけ同じ handle を rewind してください。in-place mutation または atomic replacement を検知した場合は snapshot を破棄し、prepass と main indexing pass が乖離しないよう1回だけ再認可・再openします。UTF-8 / UTF-16、NUL 拒否、growth、cancellation、lexical boundary を維持してください。576 KiB の semantic-negative/positive allocation guard は各 probe を12回実行して current-thread allocation を4 KiB未満に保ち、content 全体 mask の再導入を防ぎます。 +- `IndexCommandRunnerTests`、`FileIndexerTests`、`PerformanceTests` は `CSharpStaticInterfacePrepass` のテキスト判定、raw-byte、chunked raw-token、streaming file 契約 probe も扱います。安定した候補読み取りは各 file を1回だけ認可・openし、raw-negative はEOF growth readを行わないinitial-length bounded pass 1回に留め、raw-positive の full decode だけ同じ handle を rewindしてください。attempt 0のfull decodeもinitial lengthで停止し、`ReadByte` probeを行いません。in-place mutation または atomic replacement を検知した場合は snapshot を破棄し、prepass と main indexing pass が乖離しないよう1回だけ再認可・再openし、bounded EOF scanはretryだけが行います。UTF-8 / UTF-16、NUL 拒否、growth、cancellation、lexical boundary を維持してください。576 KiB の semantic-negative/positive allocation guard は各 probe を12回実行して current-thread allocation を4 KiB未満に保ち、content 全体 mask の再導入を防ぎます。 - parallel C# static-interface full-scan fixture は64個のimplementation fileを使い、workspace lookup buildが1回であることをperformance contractとします。CLI full scan、scoped update、MCP indexingを横断してcontract lookupをimmutable prepass snapshotに保持し、C# fileごとの再構築を戻さないでください。 - `Run_FullScan_PostPrepassCsharpContractLeavesReadinessPartialUntilCleanRetry` は fresh、rebuild、incremental-existing の各 route で full-scan extraction state が単調に維持されることを固定します。Python を C# より先に処理する fixture で、後段の C# workspace snapshot drift 後も先行 raw chunk が保存され、standard / trigram FTS の両方から検索でき、bulk-load optimization が厳密に1回であることを証明してください。 - `Run_FullScan_FatalParallelResultKeepsWorkerResourcesAliveUntilPeersStop` は一方の C# symbols worker を block し、peer に fatal extraction stall を返させます。command が速やかに戻ること、block 中の peer を release する前に worker completion と artifact-cache clear のどちらも起きないことを assertion し、process-wide hook の復元や fixture 削除の前に両 cleanup signal を待ってください。 diff --git a/changelog.d/unreleased/+initial-full-index-content-snapshot.changed.md b/changelog.d/unreleased/+initial-full-index-content-snapshot.changed.md new file mode 100644 index 0000000000..88f16cc1c6 --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-content-snapshot.changed.md @@ -0,0 +1,15 @@ +--- +category: changed +affected: + - src/CodeIndex/Indexer/Scanning/FileContentLoader.RawBytes.cs + - src/CodeIndex/Indexer/Scanning/FileContentLoader.UnknownLanguage.cs + - src/CodeIndex/Indexer/Scanning/FileContentLoader.cs +--- + +## English + +- **Stable cold-index file reads no longer probe past their initial snapshot** — language-common content loads, negative raw-token scans, the C# static-interface prepass, and unknown-language coverage stop at the initial handle length on their first attempt and validate the final handle metadata plus actual bytes read instead of issuing a redundant EOF or `ReadByte` growth probe. Mutated files still reopen once for a bounded EOF retry, over-limit growth still fails on the current handle, and positive raw-token matches remain conservative. + +## 日本語 + +- **cold index時のstableなfile readがinitial snapshotを越えてprobeしなくなりました** — 言語共通content load、negative raw-token scan、C# static-interface prepass、unknown-language coverageは、初回attemptをinitial handle lengthで停止し、冗長なEOFまたは`ReadByte` growth probeの代わりにfinal handle metadataと実読byte数を検証します。mutation時は従来どおり1回だけ再openしてbounded EOF retryを行い、上限超過growthは現在のhandleで失敗し、positive raw-token matchは保守的な判定を維持します。 diff --git a/src/CodeIndex/Indexer/Scanning/FileContentLoader.RawBytes.cs b/src/CodeIndex/Indexer/Scanning/FileContentLoader.RawBytes.cs index 2a917b61e5..d7f06159ab 100644 --- a/src/CodeIndex/Indexer/Scanning/FileContentLoader.RawBytes.cs +++ b/src/CodeIndex/Indexer/Scanning/FileContentLoader.RawBytes.cs @@ -18,18 +18,15 @@ private RawFileSnapshot ReadRawBytesWithSizeLimit( string normalizedRelativePath, CancellationToken cancellationToken) { - // Read raw bytes through a single FileStream and cap the accumulated payload at - // the configured max-file limit so a file that grew between the size probe and the read can no - // longer bypass the cap. Splitting `FileInfo.Length` from `File.ReadAllBytes` - // left a TOCTOU window where an attacker (or any build/log emitter rapidly - // appending to a generated file) could grow a 1 MB file to multi-GB between - // stat and read and force the indexer into an OOM-sized allocation; reading - // through one open handle removes the second stat call, and the read loop's - // running total guarantees we never accumulate more than the configured max-file bytes - // regardless of how aggressively a concurrent writer extends the file. - // ファイルを 1 本の FileStream で開き、設定された max-file byte 上限として累積バッファを - // 制限することで、サイズ確認と読み込みの間にファイルが肥大化しても上限を - // 回避できないようにする。 + // Read raw bytes through one FileStream. Attempt zero stops at the initial + // handle length and validates the final handle metadata instead of probing + // EOF; a changed snapshot retries once with the conventional bounded EOF + // loop. Both paths reject a final handle length over the configured cap, and + // the retry's running total prevents concurrent growth from forcing an + // unbounded allocation. + // 1本のFileStreamでraw byteを読みます。attempt 0はinitial handle lengthで停止し、 + // EOF probeの代わりにfinal handle metadataを検証します。snapshot変化時だけ従来の + // bounded EOF loopで1回retryし、どちらの経路もfinal handle lengthの上限超過を拒否します。 byte[] bytes; long sizeBytes; DateTime modifiedUtc; @@ -52,13 +49,25 @@ private RawFileSnapshot ReadRawBytesWithSizeLimit( stream, initialLength, normalizedRelativePath, + readGrowthToEnd: attempt > 0, cancellationToken); var finalSnapshot = CaptureFileHandleSnapshot(stream); modifiedUtc = finalSnapshot.ModifiedUtc; + ThrowIfReadExceedsMaxFileSize( + normalizedRelativePath, + finalSnapshot.Length); pathIdentityChanged = ReadPathIdentityChanged(absolutePath, finalSnapshot); + + if (attempt > 0 + || InitialLengthReadIsStable( + initialSnapshot, + finalSnapshot, + sizeBytes, + pathIdentityChanged)) + { + break; + } } - if ((modifiedUtc == initialSnapshot.ModifiedUtc && !pathIdentityChanged) || attempt > 0) - break; } return new RawFileSnapshot(bytes, sizeBytes, modifiedUtc); @@ -76,7 +85,7 @@ internal bool RawByteChunksMayMatch( FileIndexer.FileHandleSnapshot initialSnapshot; FileIndexer.FileHandleSnapshot finalSnapshot; bool pathIdentityChanged; - bool matched; + RawByteScanResult scan; using (var stream = OpenValidatedReadStream( absolutePath, readPath, @@ -87,26 +96,47 @@ internal bool RawByteChunksMayMatch( normalizedRelativePath, initialLength); - matched = RawByteChunksMayMatch( + scan = RawByteChunksMayMatch( stream, initialLength, normalizedRelativePath, chunkPredicate, + readGrowthToEnd: attempt > 0, cancellationToken); finalSnapshot = CaptureFileHandleSnapshot(stream); pathIdentityChanged = ReadPathIdentityChanged(absolutePath, finalSnapshot); } - if (matched) + if (scan.Matched) return true; - if ((finalSnapshot.ModifiedUtc == initialSnapshot.ModifiedUtc - && !pathIdentityChanged) - || attempt > 0) + ThrowIfReadExceedsMaxFileSize( + normalizedRelativePath, + finalSnapshot.Length); + if (attempt > 0 + || InitialLengthReadIsStable( + initialSnapshot, + finalSnapshot, + scan.BytesRead, + pathIdentityChanged)) + { return false; + } } } + private static bool InitialLengthReadIsStable( + FileIndexer.FileHandleSnapshot initialSnapshot, + FileIndexer.FileHandleSnapshot finalSnapshot, + long bytesRead, + bool pathIdentityChanged) + => bytesRead == initialSnapshot.Length + && finalSnapshot.Length == initialSnapshot.Length + && finalSnapshot.Length == bytesRead + && finalSnapshot.ModifiedUtc == initialSnapshot.ModifiedUtc + && finalSnapshot.Identity == initialSnapshot.Identity + && !pathIdentityChanged; + private FileStream OpenValidatedReadStream( string absolutePath, string expectedReadPath, @@ -160,6 +190,7 @@ private static bool ReadPathIdentityChanged( FileStream stream, long initialLength, string normalizedRelativePath, + bool readGrowthToEnd, CancellationToken cancellationToken) { var expectedLength = (int)initialLength; @@ -176,6 +207,9 @@ private static bool ReadPathIdentityChanged( } cancellationToken.ThrowIfCancellationRequested(); + if (!readGrowthToEnd) + return (bytes, total); + var extra = stream.ReadByte(); if (extra < 0) return (bytes, total); @@ -189,11 +223,14 @@ private static bool ReadPathIdentityChanged( cancellationToken); } - private bool RawByteChunksMayMatch( + private readonly record struct RawByteScanResult(bool Matched, long BytesRead); + + private RawByteScanResult RawByteChunksMayMatch( FileStream stream, long initialLength, string normalizedRelativePath, RawByteChunkPredicate chunkPredicate, + bool readGrowthToEnd, CancellationToken cancellationToken) { var buffer = ArrayPool.Shared.Rent(StreamBufferSize); @@ -209,13 +246,17 @@ private bool RawByteChunksMayMatch( 0, (int)Math.Min(buffer.Length, initialLength - total)); if (read == 0) - return false; + return new RawByteScanResult(Matched: false, total); total += read; if (chunkPredicate(buffer.AsSpan(0, read))) - return true; + return new RawByteScanResult(Matched: true, total); } + cancellationToken.ThrowIfCancellationRequested(); + if (!readGrowthToEnd) + return new RawByteScanResult(Matched: false, total); + return RawByteGrowthChunksMayMatch( stream, total, @@ -230,7 +271,7 @@ private bool RawByteChunksMayMatch( } } - private bool RawByteGrowthChunksMayMatch( + private RawByteScanResult RawByteGrowthChunksMayMatch( FileStream stream, long total, string normalizedRelativePath, @@ -246,13 +287,13 @@ private bool RawByteGrowthChunksMayMatch( 0, GetReadLengthWithinLimit(total, maxFileSizeBytes, buffer.Length)); if (read == 0) - return false; + return new RawByteScanResult(Matched: false, total); total += read; ThrowIfReadExceedsMaxFileSize(normalizedRelativePath, total); if (chunkPredicate(buffer.AsSpan(0, read))) - return true; + return new RawByteScanResult(Matched: true, total); } } diff --git a/src/CodeIndex/Indexer/Scanning/FileContentLoader.UnknownLanguage.cs b/src/CodeIndex/Indexer/Scanning/FileContentLoader.UnknownLanguage.cs index cdc0f73579..12d2eed9e4 100644 --- a/src/CodeIndex/Indexer/Scanning/FileContentLoader.UnknownLanguage.cs +++ b/src/CodeIndex/Indexer/Scanning/FileContentLoader.UnknownLanguage.cs @@ -58,10 +58,14 @@ private UnknownLanguageProbeResult ProbeUnknownLanguageCore( readPath, out var initialSnapshot); var initialLength = initialSnapshot.Length; + var readGrowthToEnd = attempt > 0; + var headerByteLimit = readGrowthToEnd + ? headerBytes.Length + : (int)Math.Min(initialLength, headerBytes.Length); var headerByteCount = FileIndexer.ReadScriptHeaderPrefix( stream, - headerBytes, + headerBytes[..headerByteLimit], cancellationToken); var language = FileIndexer.DetectLanguageFromScriptHeaderBytes( headerBytes[..headerByteCount], @@ -76,6 +80,7 @@ private UnknownLanguageProbeResult ProbeUnknownLanguageCore( if (attempt == 0 && (finalSnapshot.ModifiedUtc != initialSnapshot.ModifiedUtc || headerLengthChanged + || finalSnapshot.Identity != initialSnapshot.Identity || headerPathIdentityChanged)) { continue; @@ -100,16 +105,24 @@ private UnknownLanguageProbeResult ProbeUnknownLanguageCore( absoluteOffset: 0); var reachedEof = false; - while (prefixLength < UnknownLanguageUtf16SampleByteLimit) + while (prefixLength < UnknownLanguageUtf16SampleByteLimit + && (readGrowthToEnd || total < initialLength)) { cancellationToken.ThrowIfCancellationRequested(); + var requestedReadLength = UnknownLanguageUtf16SampleByteLimit - prefixLength; + if (!readGrowthToEnd) + { + requestedReadLength = (int)Math.Min( + requestedReadLength, + initialLength - total); + } var read = stream.Read( coverageBuffer, prefixLength, GetReadLengthWithinLimit( total, maxFileSizeBytes, - UnknownLanguageUtf16SampleByteLimit - prefixLength)); + requestedReadLength)); if (read == 0) { reachedEof = true; @@ -126,14 +139,21 @@ private UnknownLanguageProbeResult ProbeUnknownLanguageCore( } var readBuffer = coverageBuffer.AsSpan(UnknownLanguageUtf16SampleByteLimit); - while (!reachedEof) + while (!reachedEof && (readGrowthToEnd || total < initialLength)) { cancellationToken.ThrowIfCancellationRequested(); + var requestedReadLength = readBuffer.Length; + if (!readGrowthToEnd) + { + requestedReadLength = (int)Math.Min( + requestedReadLength, + initialLength - total); + } var read = stream.Read( readBuffer[..GetReadLengthWithinLimit( total, maxFileSizeBytes, - readBuffer.Length)]); + requestedReadLength)]); if (read == 0) break; @@ -145,13 +165,18 @@ private UnknownLanguageProbeResult ProbeUnknownLanguageCore( ThrowIfReadExceedsMaxFileSize(normalizedRelativePath, total); } + cancellationToken.ThrowIfCancellationRequested(); var finalSnapshot = CaptureFileHandleSnapshot(stream); - var lengthChanged = finalSnapshot.Length != initialLength || total != initialLength; + ThrowIfReadExceedsMaxFileSize( + normalizedRelativePath, + finalSnapshot.Length); var pathIdentityChanged = ReadPathIdentityChanged(absolutePath, finalSnapshot); if (attempt == 0 - && (finalSnapshot.ModifiedUtc != initialSnapshot.ModifiedUtc - || lengthChanged - || pathIdentityChanged)) + && !InitialLengthReadIsStable( + initialSnapshot, + finalSnapshot, + total, + pathIdentityChanged)) { continue; } diff --git a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs index 6210300543..167a340602 100644 --- a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs +++ b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs @@ -87,7 +87,7 @@ internal string LoadNormalizedContentForPrepass( var readPath = _resolveFileReadPath(absolutePath); cancellationToken.ThrowIfCancellationRequested(); byte[]? bytes = null; - bool lengthChanged; + long bytesRead; bool pathIdentityChanged; FileIndexer.FileHandleSnapshot initialSnapshot; FileIndexer.FileHandleSnapshot finalSnapshot; @@ -102,7 +102,7 @@ internal string LoadNormalizedContentForPrepass( initialLength); var probe = CSharpStaticInterfacePrepass.CreateRawByteContractProbe(); - var rawCandidate = RawByteChunksMayMatch( + var scan = RawByteChunksMayMatch( stream, initialLength, normalizedRelativePath, @@ -110,27 +110,34 @@ internal string LoadNormalizedContentForPrepass( ? probe .AppendAndCheckWorkspaceOrQualifiedMemberAccessCandidate : probe.AppendAndCheckWorkspaceCandidate, + readGrowthToEnd: !retryOnMutation, cancellationToken); - if (rawCandidate) + bytesRead = scan.BytesRead; + if (scan.Matched) { cancellationToken.ThrowIfCancellationRequested(); stream.Seek(0, SeekOrigin.Begin); - (bytes, _) = ReadStreamBytesWithKnownInitialLength( + (bytes, bytesRead) = ReadStreamBytesWithKnownInitialLength( stream, initialLength, normalizedRelativePath, + readGrowthToEnd: !retryOnMutation, cancellationToken); } finalSnapshot = CaptureFileHandleSnapshot(stream); - lengthChanged = finalSnapshot.Length != initialLength; + ThrowIfReadExceedsMaxFileSize( + normalizedRelativePath, + finalSnapshot.Length); pathIdentityChanged = ReadPathIdentityChanged(absolutePath, finalSnapshot); } if (retryOnMutation - && (finalSnapshot.ModifiedUtc != initialSnapshot.ModifiedUtc - || lengthChanged - || pathIdentityChanged)) + && !InitialLengthReadIsStable( + initialSnapshot, + finalSnapshot, + bytesRead, + pathIdentityChanged)) { return (null, RequiresRetry: true); } diff --git a/tests/CodeIndex.Tests/FileIndexerContentLoadingTests.cs b/tests/CodeIndex.Tests/FileIndexerContentLoadingTests.cs index f995aea072..4c8b89da8d 100644 --- a/tests/CodeIndex.Tests/FileIndexerContentLoadingTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerContentLoadingTests.cs @@ -108,9 +108,19 @@ public void FileContentLoader_StableReadPathsCaptureExactlyTwoHandleSnapshots( _ => "class Fixture { }\n", }; var path = TestProjectHelper.WriteTextFile(project.Root, "subject", source); + var openCount = 0; var snapshotCount = 0; + CountingCSharpPrepassFileStream? openedStream = null; var loader = new FileContentLoader( FileIndexer.DefaultMaxFileSizeBytes, + openReadForIndexContent: candidate => + { + openCount++; + openedStream = new CountingCSharpPrepassFileStream( + candidate, + maxReadBytes: 3); + return openedStream; + }, resolveFileReadPath: readPathShape == "load-bound" ? static candidate => Path.GetFullPath(candidate) : null, @@ -178,7 +188,299 @@ public void FileContentLoader_StableReadPathsCaptureExactlyTwoHandleSnapshots( null); } + Assert.Equal(1, openCount); Assert.Equal(2, snapshotCount); + Assert.NotNull(openedStream); + Assert.Equal(0, openedStream.ReadByteCallCount); + Assert.Equal(0, openedStream.ZeroByteReadCount); + } + + [Theory] + [InlineData("load")] + [InlineData("raw-negative")] + [InlineData("csharp-negative")] + [InlineData("unknown-coverage")] + public void FileContentLoader_SameMtimeGrowthRetriesFromFinalHandleLength( + string readPathShape) + { + using var project = TestProjectHelper.CreateTempProjectScope( + "cdidx_initial_length_growth_retry"); + var source = readPathShape switch + { + "csharp-negative" => "public class C { int M() => 0; }\n" + new string('x', 64), + "unknown-coverage" => "plain unknown-language coverage\n" + new string('x', 64), + _ => "class Fixture { }\n" + new string('x', 64), + }; + const string growth = "tail-growth\n"; + var path = TestProjectHelper.WriteTextFile(project.Root, "subject", source); + File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddMinutes(-3)); + var stableModifiedUtc = File.GetLastWriteTimeUtc(path); + var openCount = 0; + var snapshotCount = 0; + var openedStreams = new List(); + var loader = new FileContentLoader( + FileIndexer.DefaultMaxFileSizeBytes, + openReadForIndexContent: candidate => + { + openCount++; + Action? afterFirstRead = null; + if (openCount == 1) + { + afterFirstRead = () => + { + File.AppendAllText(path, growth); + File.SetLastWriteTimeUtc(path, stableModifiedUtc); + }; + } + + var stream = new CountingCSharpPrepassFileStream( + candidate, + maxReadBytes: 7, + afterFirstRead); + openedStreams.Add(stream); + return stream; + }, + fileHandleSnapshotCapturedForTesting: () => snapshotCount++); + + switch (readPathShape) + { + case "load": + Assert.Equal(source + growth, loader.Load( + path, + "subject", + "subject", + CancellationToken.None).Content); + break; + case "raw-negative": + Assert.False(loader.RawByteChunksMayMatch( + path, + "subject", + static _ => false, + CancellationToken.None)); + break; + case "csharp-negative": + var first = loader.LoadCSharpStaticInterfaceCandidateContentForPrepass( + path, + "subject", + "subject", + retryOnMutation: true, + includeQualifiedMemberAccessCandidate: false, + includeChecksum: false, + CancellationToken.None); + Assert.True(first.RequiresRetry); + var second = loader.LoadCSharpStaticInterfaceCandidateContentForPrepass( + path, + "subject", + "subject", + retryOnMutation: false, + includeQualifiedMemberAccessCandidate: false, + includeChecksum: false, + CancellationToken.None); + Assert.False(second.RequiresRetry); + Assert.Null(second.Content); + break; + case "unknown-coverage": + var unknown = loader.ProbeUnknownLanguage( + path, + "subject", + "subject", + CancellationToken.None); + Assert.Equal( + FileIndexer.FileProbeStatus.Unsupported, + unknown.LanguageDetection.Status); + Assert.True(unknown.IsCoverageCandidate); + break; + default: + throw new ArgumentOutOfRangeException( + nameof(readPathShape), + readPathShape, + null); + } + + Assert.Equal(2, openCount); + Assert.Equal(4, snapshotCount); + Assert.Equal(0, openedStreams[0].ReadByteCallCount); + Assert.Equal(0, openedStreams[0].ZeroByteReadCount); + } + + [Fact] + public void FileContentLoader_Load_ShrinkThenRegrowRetriesAndReadsRegrownSnapshot() + { + using var project = TestProjectHelper.CreateTempProjectScope( + "cdidx_initial_length_shrink_regrow"); + var originalSource = new string('a', 16 * 1024) + "\n"; + var regrownSource = new string('b', 16 * 1024) + "\n"; + var path = TestProjectHelper.WriteTextFile(project.Root, "subject", originalSource); + File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddMinutes(-3)); + var stableModifiedUtc = File.GetLastWriteTimeUtc(path); + var openCount = 0; + var snapshotCount = 0; + var openedStreams = new List(); + var loader = new FileContentLoader( + FileIndexer.DefaultMaxFileSizeBytes, + openReadForIndexContent: candidate => + { + openCount++; + if (openCount == 2) + { + File.WriteAllText(path, regrownSource); + File.SetLastWriteTimeUtc(path, stableModifiedUtc); + } + + Action? afterFirstRead = openCount == 1 + ? () => + { + File.WriteAllText(path, "short\n"); + File.SetLastWriteTimeUtc(path, stableModifiedUtc); + } + : null; + var stream = new CountingCSharpPrepassFileStream( + candidate, + maxReadBytes: 4 * 1024, + afterFirstRead); + openedStreams.Add(stream); + return stream; + }, + fileHandleSnapshotCapturedForTesting: () => snapshotCount++); + + var loaded = loader.Load( + path, + "subject", + "subject", + CancellationToken.None); + + Assert.Equal(regrownSource, loaded.Content); + Assert.Equal(2, openCount); + Assert.Equal(4, snapshotCount); + Assert.Equal(0, openedStreams[0].ReadByteCallCount); + } + + [Theory] + [InlineData("load")] + [InlineData("raw-negative")] + public void FileContentLoader_GrowthBeyondLimitUsesFinalHandleSnapshotWithoutRetry( + string readPathShape) + { + using var project = TestProjectHelper.CreateTempProjectScope( + "cdidx_initial_length_growth_cap"); + var path = TestProjectHelper.WriteTextFile( + project.Root, + "subject", + new string('x', 128)); + var initialLength = new FileInfo(path).Length; + var openCount = 0; + var snapshotCount = 0; + CountingCSharpPrepassFileStream? openedStream = null; + var loader = new FileContentLoader( + initialLength + 8, + openReadForIndexContent: candidate => + { + openCount++; + openedStream = new CountingCSharpPrepassFileStream( + candidate, + maxReadBytes: 7, + afterFirstRead: () => File.AppendAllText(path, new string('y', 32))); + return openedStream; + }, + fileHandleSnapshotCapturedForTesting: () => snapshotCount++); + + var exception = Assert.Throws(() => + { + if (readPathShape == "load") + { + loader.Load(path, "subject", "subject", CancellationToken.None); + return; + } + + loader.RawByteChunksMayMatch( + path, + "subject", + static _ => false, + CancellationToken.None); + }); + + Assert.Contains("grew during read", exception.Message, StringComparison.Ordinal); + Assert.Equal(1, openCount); + Assert.Equal(2, snapshotCount); + Assert.NotNull(openedStream); + Assert.Equal(0, openedStream.ReadByteCallCount); + Assert.Equal(0, openedStream.ZeroByteReadCount); + } + + [Fact] + public void FileContentLoader_RawPositiveGrowthRemainsConservativeWithoutRetry() + { + using var project = TestProjectHelper.CreateTempProjectScope( + "cdidx_initial_length_raw_positive"); + var path = TestProjectHelper.WriteTextFile( + project.Root, + "subject", + new string('x', 128)); + var initialLength = new FileInfo(path).Length; + var openCount = 0; + var loader = new FileContentLoader( + initialLength + 8, + openReadForIndexContent: candidate => + { + openCount++; + return new CountingCSharpPrepassFileStream( + candidate, + maxReadBytes: 7, + afterFirstRead: () => File.AppendAllText(path, new string('y', 32))); + }); + + Assert.True(loader.RawByteChunksMayMatch( + path, + "subject", + static bytes => !bytes.IsEmpty, + CancellationToken.None)); + Assert.Equal(1, openCount); + } + + [Fact] + public void FileContentLoader_Load_SecondAttemptGrowthUsesBoundedEofRead() + { + using var project = TestProjectHelper.CreateTempProjectScope( + "cdidx_initial_length_second_growth"); + var path = TestProjectHelper.WriteTextFile( + project.Root, + "subject", + new string('x', 1024)); + var initialLength = new FileInfo(path).Length; + File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddMinutes(-3)); + var stableModifiedUtc = File.GetLastWriteTimeUtc(path); + var openCount = 0; + var snapshotCount = 0; + var openedStreams = new List(); + var loader = new FileContentLoader( + initialLength + 16, + openReadForIndexContent: candidate => + { + openCount++; + Action afterFirstRead = openCount == 1 + ? () => + { + File.AppendAllText(path, "tail"); + File.SetLastWriteTimeUtc(path, stableModifiedUtc); + } + : () => File.AppendAllText(path, new string('y', 32)); + var stream = new CountingCSharpPrepassFileStream( + candidate, + maxReadBytes: 64, + afterFirstRead); + openedStreams.Add(stream); + return stream; + }, + fileHandleSnapshotCapturedForTesting: () => snapshotCount++); + + var exception = Assert.Throws(() => + loader.Load(path, "subject", "subject", CancellationToken.None)); + + Assert.Contains("grew during read", exception.Message, StringComparison.Ordinal); + Assert.Equal(2, openCount); + Assert.Equal(3, snapshotCount); + Assert.Equal(0, openedStreams[0].ReadByteCallCount); + Assert.Equal(1, openedStreams[1].ReadByteCallCount); } [Fact] diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 653d83885d..ea35795d4f 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -2214,6 +2214,8 @@ public void LoadCSharpStaticInterfaceCandidateContentForPrepass_ProbeShapesUseOn Assert.Equal(1, openCount); Assert.Equal(1, authorizationCount); Assert.NotNull(openedStream); + Assert.Equal(0, openedStream.ReadByteCallCount); + Assert.Equal(0, openedStream.ZeroByteReadCount); Assert.Equal(expectsRawCandidate, candidateContent is not null); Assert.Equal( expectsSemanticContract, @@ -9124,7 +9126,9 @@ internal CountingCSharpPrepassFileStream( internal long BytesRead { get; private set; } internal long RawProbeBytes { get; private set; } + internal int ReadByteCallCount { get; private set; } internal int RewindCount { get; private set; } + internal int ZeroByteReadCount { get; private set; } public override int Read(byte[] buffer, int offset, int count) { @@ -9142,6 +9146,7 @@ public override int Read(Span buffer) public override int ReadByte() { + ReadByteCallCount++; var value = base.ReadByte(); if (value >= 0) BytesRead++; @@ -9161,6 +9166,8 @@ public override long Seek(long offset, SeekOrigin origin) private void RecordRead(int read) { + if (read == 0) + ZeroByteReadCount++; BytesRead += read; if (read <= 0 || _firstReadObserved) return; From bc96791a0afad51348e153a30dc19182a412454f Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 30 Aug 2026 00:45:15 +0900 Subject: [PATCH 09/11] Materialize fresh reference source lookups --- DEVELOPER_GUIDE.md | 6 +- TESTING_GUIDE.md | 8 +- ...l-index-reference-source-lookup.changed.md | 17 + .../DbWriter.AuthoritativeFreshBulkInsert.cs | 19 +- ...AuthoritativeFreshReferenceSourceLookup.cs | 199 +++++++ .../Database/DbWriter.ReferenceSql.cs | 22 +- src/CodeIndex/Database/DbWriter.References.cs | 12 +- src/CodeIndex/Database/DbWriter.cs | 4 +- .../AuthoritativeFreshRawBulkInsertTests.cs | 505 ++++++++++++++++++ tests/CodeIndex.Tests/DatabaseTests.cs | 6 + .../FreshReferenceResolutionTests.cs | 41 ++ tests/CodeIndex.Tests/PerformanceTests.cs | 9 +- 12 files changed, 829 insertions(+), 19 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-reference-source-lookup.changed.md create mode 100644 src/CodeIndex/Database/DbWriter.AuthoritativeFreshReferenceSourceLookup.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 024c3e1150..bbf239c040 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1476,7 +1476,8 @@ Current stable codes and triggers: | Caller-owned write batching | Full-scan and other atomic file writes already run inside one caller-owned transaction, so their language-neutral chunk, symbol, issue, reference-line, and reference inserts cap each statement at 32 parameters. Every batch uses compact, one-origin SQLite numeric slots (`?1` through `?N`) in row/column order; this reduces parameter-name resolution work while preserving the existing statement-size, cancellation, and checkpoint contracts. For operations above 500 rows, persistent `db_writer_batch_checkpoint` records are emitted only when progress crosses a 500-row boundary and at completion, avoiding a synchronous log flush for every tiny statement. Public writer APIs retain the SQLite-variable-limit batch shape and their existing per-batch transaction/SAVEPOINT contract. | | Prepared savepoint controls | `DbWriter` leases only a fixed, bounded set of control statements from the connection's prepared-command cache: the first nested `sp_1` SAVEPOINT / RELEASE / ROLLBACK trio used by per-file full-index scopes, the atomic metadata savepoints, and the FTS bulk-load owner savepoint. Every lease rebinds the current outer `SqliteTransaction`; a cacheless writer still creates and disposes one command per call. Depth-two and deeper savepoint names remain dynamic and bypass the cache, and cancellation, rollback, terminal-state, and transaction-gate contracts are unchanged. | | Authoritative-fresh raw insert scope | After the empty-database CLI path revalidates its authoritative-fresh claim inside the caller-owned transaction, only the extraction pipeline's new-file, chunk, symbol, new-file issue, fresh reference-line, and atomic fresh-reference INSERTs may bind and execute through SQLitePCLRaw on the provider-owned connection handle. These native positional bindings use a separate 512-parameter ceiling while provider-backed caller-owned writes retain their 32-parameter limit. Fresh file and reference-line writes are DONE-only: file insertion captures the same connection's positive `last_insert_rowid`, while every reference-line batch reads the greater of `MAX(id)` and `sqlite_sequence.seq`, checks the complete Int64 range, and inserts explicit contiguous IDs with `?1 + input_ordinal`. Reading the floor for every batch preserves AUTOINCREMENT history and observes inserts between batches without retaining rollback-sensitive allocator state. Invalid floors and identity-range overflow fail before the INSERT executes. Every executed fresh identity write validates `sqlite3_changes()` before publishing IDs; a row-count mismatch, constraint, cleanup failure, or cancellation discards the affected prepared statement while the caller's per-file savepoint owns data rollback. The scope preserves exact tail/write-count shapes, batch hooks, row-skip replay, and outer transaction atomicity; a 32-entry LRU retains recurring full and tail statement shapes. The synchronous full-scan persistence consumer and `DbWriter` transaction-owner check keep the non-thread-safe cache single-owner. Every lease resets and clears bindings, errors retain the original step result, cancellation maps SQLite interrupt to `OperationCanceledException`, and all cached statements are finalized before graph/index/FTS work. During this same transaction the three `files_resource_generation_*` triggers are suspended, then recreated only after native statements finalize; the resource-list generation advances exactly once when at least one file was persisted and stays unchanged for an empty repository. Rollback restores both schema and generation atomically. Replacement, incremental, rebuild, symbols-only, fresh-claim race fallback, MCP, and public writer paths remain on Microsoft.Data.Sqlite, retain their established `RETURNING` behavior, and keep per-mutation generation invalidation. | -| Authoritative-fresh core secondary indexes | The same revalidated empty-database CLI transaction drops 22 language-neutral secondary indexes on `files`, `chunks`, `file_issues`, and `symbols` before persistence, then builds each B-tree once after every native INSERT statement has finalized and before graph or readiness queries begin. UNIQUE autoindexes remain active, so path and table constraints keep their normal enforcement. `idx_symbols_file` also remains active because fresh-reference insertion resolves each containing source symbol through a correlated per-file lookup; dropping it would turn that hot path into repeated full symbol-table scans. Cancellation or failure leaves restoration to the outer rollback, which atomically restores the pre-load schema; rebuild, incremental, fresh-claim race fallback, and MCP paths retain the indexes throughout their writes. Canonical DDL is shared by schema initialization, opportunistic read migration, and the bulk-load guard so the deferred set cannot drift from the completed database contract. | +| Authoritative-fresh source-symbol lookup | Before native statements are prepared, the raw scope creates a connection-local TEMP `WITHOUT ROWID` snapshot with partial indexes for folded name, folded display name, and legacy ASCII `NOCASE` fallback. Each atomic reference collection clears that snapshot and copies the symbols for its distinct source file IDs once through `idx_symbols_file`; references without a container skip materialization. The three indexed probes use `UNION` to preserve the existing name-or-display semantics when one symbol matches more than one branch, then retain the same containing-range and innermost-span/start/id ranking. TEMP schema creation belongs to the caller's outer transaction, while each file savepoint owns its snapshot population and reference writes, so cancellation, failure, and rollback restore both together. Provider-backed, rebuild, incremental, fresh-claim fallback, MCP, and public-writer paths neither create nor query the TEMP table. | +| Authoritative-fresh core secondary indexes | The same revalidated empty-database CLI transaction drops 22 language-neutral secondary indexes on `files`, `chunks`, `file_issues`, and `symbols` before persistence, then builds each B-tree once after every native INSERT statement has finalized and before graph or readiness queries begin. UNIQUE autoindexes remain active, so path and table constraints keep their normal enforcement. `idx_symbols_file` also remains active because fresh-reference insertion copies each relevant file's symbols into its indexed TEMP snapshot once; dropping it would turn every per-file materialization into a full symbol-table scan. Cancellation or failure leaves restoration to the outer rollback, which atomically restores the pre-load schema; rebuild, incremental, fresh-claim race fallback, and MCP paths retain the indexes throughout their writes. Canonical DDL is shared by schema initialization, opportunistic read migration, and the bulk-load guard so the deferred set cannot drift from the completed database contract. | | Checkpointing | `DbWriter` runs `PRAGMA wal_checkpoint(PASSIVE)` after each outer transaction commit, and SQLite may also checkpoint automatically after the configured 1000-page threshold. Both checkpoint paths are opportunistic: active readers are not blocked, and an uncheckpointed WAL is expected state rather than corruption. | | Checkpoint result contract | Explicit `PRAGMA wal_checkpoint(TRUNCATE)` paths execute a reader and return a structured result containing SQLite's `(busy, log, checkpointed)` values. Non-zero `busy` or positive remaining pages is unsuccessful with a bounded machine reason. `(0, -1, -1)` is SQLite's successful non-WAL no-op. Instance checkpointing, the static read-only-fallback preflight, query diagnostics, top-level status, and nested connection-policy status preserve the same result and counts. Raw exception text and paths must not enter diagnostics. | | Crash recovery | If the process is killed after SQLite has committed a transaction but before checkpointing, the next normal opener rolls the WAL forward; no manual recovery step is required. If the process dies before a transaction commits, SQLite rolls that transaction back. | @@ -5625,7 +5626,8 @@ apply 時は `PRAGMA optimize` を実行します。 | caller-owned write batch | full-scan などの atomic file write は既に1つの caller-owned transaction 内で実行されるため、言語共通の chunk、symbol、issue、reference-line、reference insert は statement を32 parameter以下に制限します。すべてのbatchはrow / column順にcompactな1-origin SQLite numeric slot（`?1`〜`?N`）を使い、既存のstatement-size、cancellation、checkpoint契約を保ったままparameter name解決の処理を抑えます。500 rowを超えるoperationでは、永続 `db_writer_batch_checkpoint` を500 row境界をまたいだ時点と完了時だけ出力することで、小さなstatementごとの同期log flushを避けます。public writer API は SQLite variable limit までの batch 形状と既存の batch ごとの transaction / SAVEPOINT 契約を維持します。 | | prepared savepoint control | `DbWriter` は connection の prepared-command cache から固定・有界な control statement だけを借ります。対象は file 単位 full-index scope が使う最初の nested `sp_1` の SAVEPOINT / RELEASE / ROLLBACK、atomic metadata savepoint、FTS bulk-load owner savepoint です。各 lease は現在の outer `SqliteTransaction` へ再 bind し、cache なし writer は従来どおり呼び出しごとに command を作成・破棄します。depth 2 以深の savepoint 名は動的なまま cache を迂回し、cancellation、rollback、terminal state、transaction gate の契約は変更しません。 | | authoritative-fresh raw insert scope | empty-database CLI経路がcaller-owned transaction内でauthoritative-fresh claimを再検証した後に限り、extraction pipelineのnew-file、chunk、symbol、new-file issue、fresh reference-line、atomic fresh-reference INSERTをprovider所有connection handle上のSQLitePCLRawでbind / executeします。native positional bindingは専用の512 parameter上限を使い、provider経由のcaller-owned writeは32 parameter上限を維持します。fresh file / reference-line writeもDONE-onlyです。file insertは同じconnectionの正の`last_insert_rowid`を取得し、reference-line batchは毎回`MAX(id)`と`sqlite_sequence.seq`の大きい方を読み、Int64範囲全体を検証して`?1 + input_ordinal`の明示的な連続IDを挿入します。batchごとのfloor読取により、rollback依存のallocator stateを保持せずAUTOINCREMENT履歴とbatch間insertを反映します。不正floorとidentity range overflowはINSERT実行前に失敗します。実行済みfresh identity writeはID公開前に`sqlite3_changes()`を検証し、row count不一致、constraint、cleanup failure、cancellationでは対象prepared statementを破棄し、data rollbackはcallerのfile単位SAVEPOINTが所有します。scopeは正確なtail / write-count形状、batch hook、row-skip replay、outer transaction atomicityを維持し、32-entry LRUで繰り返すfull / tail statement形状を保持します。同期的なfull-scan persistence consumerと`DbWriter`のtransaction owner検査により、非thread-safe cacheはsingle-ownerのままです。各leaseはresetとbinding clearを行い、error時は元のstep結果を保持し、SQLite interruptを`OperationCanceledException`へ変換し、graph / index / FTS処理より前に全cached statementをfinalizeします。同じtransaction内では3本の`files_resource_generation_*` triggerを停止し、native statementのfinalize後だけ再作成します。fileを1件以上永続化した場合はresource-list generationを厳密に1回進め、空repositoryでは変更しません。rollback時はschemaとgenerationを一括で元へ戻します。replacement、incremental、rebuild、symbols-only、fresh-claim race fallback、MCP、public writer経路はMicrosoft.Data.Sqlite、既存の`RETURNING`挙動、mutationごとのgeneration invalidationを維持します。 | -| authoritative-fresh core secondary index | 同じempty-database CLI transactionがauthoritative-fresh claimを再検証した後、`files`、`chunks`、`file_issues`、`symbols`の言語共通secondary index 22本をpersistence前に停止し、全native INSERT statementのfinalize後かつgraph / readiness queryの開始前に各B-treeを1回だけ構築します。UNIQUE autoindexは維持するため、pathとtable constraintは通常どおり適用されます。fresh-reference insertが相関するfile単位lookupでsource symbolを解決するため、`idx_symbols_file`も維持し、このhot pathがsymbol table全体の反復scanへ退行しないようにします。cancel / failure時はouter rollbackがload前のschemaをatomicに復元し、rebuild、incremental、fresh-claim race fallback、MCP経路はwrite中もindexを維持します。canonical DDLをschema initialization、opportunistic read migration、bulk-load guardで共有し、deferred setと完了DBの契約がずれないようにします。 | +| authoritative-fresh source-symbol lookup | native statementをprepareする前に、raw scopeはfold済みname、fold済みdisplay name、legacy ASCII `NOCASE` fallback用partial indexを備えたconnection-local TEMP `WITHOUT ROWID` snapshotを作成します。atomic reference collectionごとにsnapshotをclearし、異なるsource file IDのsymbolを`idx_symbols_file`経由で各1回copyします。containerを持たないreferenceではmaterializationを省きます。3本のindexed probeは、同じsymbolが複数branchに一致する場合も既存のname-or-display semanticsを保つため`UNION`で重複を除き、その後もcontaining rangeとinnermost span / start / idの同じrankingを維持します。TEMP schema作成はcallerのouter transactionに属し、各file savepointがsnapshot populationとreference writeを一緒に所有するため、cancel、failure、rollbackでは両方を復元します。provider、rebuild、incremental、fresh-claim fallback、MCP、public-writer経路はTEMP tableを作成も参照もしません。 | +| authoritative-fresh core secondary index | 同じempty-database CLI transactionがauthoritative-fresh claimを再検証した後、`files`、`chunks`、`file_issues`、`symbols`の言語共通secondary index 22本をpersistence前に停止し、全native INSERT statementのfinalize後かつgraph / readiness queryの開始前に各B-treeを1回だけ構築します。UNIQUE autoindexは維持するため、pathとtable constraintは通常どおり適用されます。fresh-reference insertが関連する各fileのsymbolをindexed TEMP snapshotへ1回copyするため、`idx_symbols_file`も維持し、file単位materializationがsymbol table全体のscanへ退行しないようにします。cancel / failure時はouter rollbackがload前のschemaをatomicに復元し、rebuild、incremental、fresh-claim race fallback、MCP経路はwrite中もindexを維持します。canonical DDLをschema initialization、opportunistic read migration、bulk-load guardで共有し、deferred setと完了DBの契約がずれないようにします。 | | checkpoint | `DbWriter` は outer transaction commit 後に `PRAGMA wal_checkpoint(PASSIVE)` を実行し、SQLite も設定済みの 1000 page threshold を超えると自動 checkpoint する場合があります。どちらの checkpoint path も opportunistic で、active reader は block されず、未 checkpoint の WAL は corruption ではなく期待される状態です。 | | checkpoint result contract | 明示的な `PRAGMA wal_checkpoint(TRUNCATE)` path は reader を実行し、SQLite の `(busy, log, checkpointed)` を含む構造化結果を返します。`busy` が 0 以外、または remaining page が正の場合は、上限付き machine reason を伴う unsuccessful result です。`(0, -1, -1)` は SQLite の非 WAL database に対する成功 no-op です。instance checkpoint、read-only fallback 前の static preflight、query diagnostics、top-level status、nested connection-policy status は同じ結果と count を保持します。raw exception text や path を diagnostics に含めてはいけません。 | | crash recovery | SQLite が transaction を commit した後、checkpoint 前に process が kill された場合、次の通常 open が WAL を roll forward するため手動 recovery は不要です。commit 前に process が終了した transaction は SQLite により rollback されます。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 342420b075..7c0fbc3550 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -777,10 +777,10 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result Concurrent read and read-during-write scenarios (WAL mode validation), including the issue #180 bug-catching snapshot-isolation regressions for all three multi-statement reader entry points: (1) `GetStatus` seeds `refs == files * refsPerFile` and asserts every concurrent observation preserves that invariant; (2) `AnalyzeSymbol` seeds one symbol `S` plus matching reference/caller pairs, toggles a second file symmetrically, and asserts `references.Count == callers.Count` across every `inspect`/`analyze_symbol` bundle; (3) `GetRepoMap` seeds a baseline modified timestamp and toggles a newer file, asserting `latest_modified == workspace_latest_modified` across every map call. Each test fails without the DEFERRED-transaction wrap on the matching reader and passes with it. - `PerformanceTests.cs` Bounded CI smoke coverage plus large-scale data benchmarks. `CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` and the allocation budget guards run in the default `net8.0` suite, so they are blocking PR/CI checks on the production target, but their broad budgets are intended to catch only severe indexing/search or allocation regressions rather than act as benchmarks. `ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` uses dense C# private-property receivers and Python imported-type calls to prevent per-candidate full-symbol rescans from returning. `ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` covers dense C# declaration containers and GitHub Actions jobs so name/range ownership resolution stays indexed. `Extraction_DenseDelimitedLists_StayWithinAllocationBudget` covers Python imports, YAML needs, JSON paths, and Fortran procedure lists without temporary split-array growth. `ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` keeps all-language dedupe identities value-based when qualified names are long. Large-scale manual tests remain skip-by-default; run a selected test on the production target with `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter `. - `AuthoritativeFreshRawBulkInsertTests` fixes the raw-scope activation and single-owner transaction guard, exact 512-parameter/tail/write-count shapes, 32-entry bounded LRU behavior, Unicode including embedded NUL, stable dates, NULL and 64-bit integer bindings, provider-only routing exclusions, constraint row-skip replay, deterministic interrupt rollback, hook-failure cleanup, finalization counts, and immediate provider reuse. Its fresh identity cases cover positive 64-bit file last-rowids, reference-line floors from both the live table and deleted AUTOINCREMENT history, duplicate-sequence fail-closed behavior, the 170/171-row boundary, inserts between batches, Int64 exhaustion before DML, changed-row mismatches, constraint/cancellation discard and reprepare, ID publication only after successful DML, and per-file savepoint rollback. The raw SQL-plan guard requires one explicit contiguous-ID INSERT without `RETURNING` or a correlated input scan; provider, rebuild, incremental, and MCP coverage retain their established `RETURNING` path. The resource-generation case inserts multiple files while the authoritative scope is active, requires one generation advance after commit, verifies an empty successful scope does not advance, verifies all three mutation triggers are restored, proves ordinary provider writes advance again, and proves an abandoned scope rolls schema, generation, and rows back together. Scope statistics mark `Completed=true` only after native statements finalize, resource triggers are restored, the conditional generation advance runs, and cancellation checks succeed; the internal completion marker is set only after the reporting hook returns, so a reporting-hook exception still makes `Complete()` fail and leaves the scope non-completed. Keep the direct tests paired with fresh/rebuild, update, claim-race fallback, MCP, mixed-language CLI, and recoverable file-failure integration assertions so only the authoritative empty CLI extraction interval observes raw work. + `AuthoritativeFreshRawBulkInsertTests` fixes the raw-scope activation and single-owner transaction guard, exact 512-parameter/tail/write-count shapes, 32-entry bounded LRU behavior, Unicode including embedded NUL, stable dates, NULL and 64-bit integer bindings, provider-only routing exclusions, constraint row-skip replay, deterministic interrupt rollback, hook-failure cleanup, finalization counts, and immediate provider reuse. Its fresh identity cases cover positive 64-bit file last-rowids, reference-line floors from both the live table and deleted AUTOINCREMENT history, duplicate-sequence fail-closed behavior, the 170/171-row boundary, inserts between batches, Int64 exhaustion before DML, changed-row mismatches, constraint/cancellation discard and reprepare, ID publication only after successful DML, and per-file savepoint rollback. The raw SQL-plan guard requires one explicit contiguous-ID INSERT without `RETURNING` or a correlated input scan; provider, rebuild, incremental, and MCP coverage retain their established `RETURNING` path. Source-symbol coverage crosses the 36-row reference batch boundary and multiple files, exercises folded name, folded display name, legacy ASCII `NOCASE`, duplicate branch matches, non-ASCII fallback rejection, and nested equal-rank ties, and requires exact source IDs while null or empty containers remain unresolved. Its plan assertions require `idx_symbols_file` during materialization plus all three TEMP partial indexes during lookup, and reject scans of either source; savepoint failure and SQLite interrupt must restore snapshot and main rows together. The provider runtime test also proves ordinary fresh resolution never creates or queries the TEMP table. The resource-generation case inserts multiple files while the authoritative scope is active, requires one generation advance after commit, verifies an empty successful scope does not advance, verifies all three mutation triggers are restored, proves ordinary provider writes advance again, and proves an abandoned scope rolls schema, generation, and rows back together. Scope statistics mark `Completed=true` only after native statements finalize, resource triggers are restored, the conditional generation advance runs, and cancellation checks succeed; the internal completion marker is set only after the reporting hook returns, so a reporting-hook exception still makes `Complete()` fail and leaves the scope non-completed. Keep the direct tests paired with fresh/rebuild, update, claim-race fallback, MCP, mixed-language CLI, and recoverable file-failure integration assertions so only the authoritative empty CLI extraction interval observes raw work. `PreparedCommandCacheTests.DbWriter_WithCache_FixedSavepointControlsReuseAcrossCommitRollbackAndTransactionRebind` pins cache misses on the first depth-one SAVEPOINT / RELEASE / ROLLBACK and cache hits after the outer transaction changes. Pair it with the deep-savepoint test, which keeps depth two and beyond outside the cache, the cancelled-nested-begin test, which leases no control command and leaves the outer scope reusable, and the metadata/FTS test, which applies the same fixed-statement contract across every atomic marker surface. `CoreSecondaryIndexBulkLoadGuardTests` pins the exact canonical set of 23 language-neutral `files` / `chunks` / `file_issues` / `symbols` secondary indexes and the 22-index deferred subset, requires a caller-owned transaction at both scope boundaries, keeps the file-path UNIQUE autoindex and reference-source `idx_symbols_file` lookup active, exercises restore over a populated table, and proves cancellation leaves DDL recovery to outer rollback. The full-scan reference-index lifecycle theory pairs this direct coverage with production ordering: only the authoritative fresh CLI run observes `dropped` then `restored`, the raw scope has finalized before restore, every graph phase sees the complete core set, and a nonempty `--rebuild` never defers it. - `PerformanceTests.AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations` alternates three measured provider/raw pairs after one warm-up over one file and 10,000 chunks, symbols, issues, and references, using the production 32-parameter provider boundaries, provider `RETURNING`, and raw DONE/explicit-identity paths. It asserts exact persisted-result parity, including file timestamp text, and reports per-stage/total elapsed time plus current-thread allocation without a wall-clock ratio gate. Run it with `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter FullyQualifiedName~AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations`. + `PerformanceTests.AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations` alternates three measured provider/raw pairs after one warm-up over one file and 10,000 chunks, symbols, issues, and references, using the production 32-parameter provider boundaries, provider `RETURNING`, and raw DONE/explicit-identity paths. Its source fixture spans the entire file so reference-source IDs are part of exact provider/raw persisted-result parity, together with file timestamp text. The test reports per-stage/total elapsed time plus current-thread allocation without a wall-clock ratio gate. Run it with `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter FullyQualifiedName~AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations`. Focused authoritative-fresh fold-readiness coverage spans `DatabaseTests`, `IndexCommandRunnerTests`, and `McpServerToolsCallTests`: the built-in empty-database CLI/MCP path must consume its claim once, retain the NULL-column verification, skip the stored-value re-fold scan, and produce the same readiness/version/fingerprint/language stamps and Unicode, Markdown, C#, Nim, and TypeScript query results as full validation. Pair it with fail-closed cases for each initially nonempty ownership table (`files`, `symbols`, or `symbol_references`), a wrong or reused claim, an intervening external commit observed through `PRAGMA data_version`, rebuild/update/legacy/public-writer paths, and custom plugins, patterns, or post-extraction hooks; full validation must still reject NULL and stale non-NULL folds. A run-barrier regression must also activate a custom producer and then reload back to built-in-only before readiness: the current producer count returns to zero, but the monotonic mutation generation changes and forces full validation. Unchanged missing-directory and diagnostic-only publications must not change that generation. A deterministic cancel-after-`BEGIN IMMEDIATE` test must prove that the raw transaction is rolled back and the same writer can immediately start and commit another transaction. For performance audits, alternate identical repository-scale fresh fixtures, isolate the readiness-finalization interval, and report elapsed time plus `GC.GetAllocatedBytesForCurrentThread`; adoption requires removing row-count-proportional managed allocation without changing rows, stamps, or query results. Keep wall-clock measurements out of blocking CI assertions and remove temporary instrumentation after recording the result. @@ -1930,9 +1930,9 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" 並行読み取りと書き込み中読み取りシナリオ（WALモード検証）。issue #180 の bug-catching な snapshot 隔離回帰テストを 3 つの multi-statement reader 経路について含む。(1) `GetStatus` は `refs == files * refsPerFile` の seed 不変条件を立て、並行観測が常にこの条件を維持することを要求する。(2) `AnalyzeSymbol` はシンボル `S` に対して reference/caller を対称に 1 対 1 で seed し、もう 1 ファイルを対称に toggle することで `inspect` / `analyze_symbol` bundle の `references.Count == callers.Count` を常に保証する。(3) `GetRepoMap` はベースラインの modified と新しい toggle 対象ファイルを用意し、`latest_modified == workspace_latest_modified` が常に一致することを要求する。各テストは対応する reader の DEFERRED transaction を外すと落ち、戻すと通ることを確認済み。 - `PerformanceTests.cs` bounded な CI smoke と大規模データベンチマークを扱います。`CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` と allocation budget guard は通常の `net8.0` suite で実行されるため production target 上の PR / CI blocking check ですが、benchmark ではなく重大な indexing/search または allocation 退行だけを拾う広めの budget を使います。`ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` は密な C# private-property receiver と Python imported-type call を使い、candidate ごとの full-symbol 再走査が戻るのを防ぎます。`ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` は密な C# declaration container と GitHub Actions job を扱い、name / range ownership 解決の索引化を維持します。`Extraction_DenseDelimitedLists_StayWithinAllocationBudget` は Python import、YAML needs、JSON path、Fortran procedure list を使い、一時 split-array の増加を防ぎます。`ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` は長い qualified name でも全言語共通 dedupe identity を value-based に維持します。大規模な手動 test は引き続きデフォルト Skip とし、production target で `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter ` を実行します。 - `AuthoritativeFreshRawBulkInsertTests` はraw scopeのactivationとsingle-owner transaction guard、正確な512 parameter / tail / write-count形状、32-entry bounded LRU、embedded NULを含むUnicode、stable date、NULL / 64-bit integer binding、provider専用routing除外、constraint時のrow-skip replay、deterministic interrupt rollback、hook failure cleanup、finalize count、直後のprovider再利用を固定します。fresh identity caseは正の64-bit file last-rowid、live tableと削除済みAUTOINCREMENT履歴の両方から得るreference-line floor、重複sequence rowのfail-closed、170/171 row境界、batch間insert、DML前のInt64枯渇、changed-row不一致、constraint / cancellation時のdiscard / reprepare、DML成功後だけのID公開、file単位SAVEPOINT rollbackを検証します。raw SQL-plan guardは`RETURNING`や相関input scanを含まない明示的な連続ID INSERTを要求し、provider、rebuild、incremental、MCP coverageは既存の`RETURNING`経路を維持します。scope statsの`Completed=true`はnative statementのfinalizeとcancellation check成功後を示しますが、内部completion markerはreporting hookの正常return後にだけ設定するため、reporting-hook exceptionでは`Complete()`が失敗しscopeもnon-completedのままです。direct testをfresh / rebuild、update、claim-race fallback、MCP、mixed-language CLI、recoverable file failureのintegration assertionと対にし、authoritative empty CLI extraction区間だけがraw workを観測することを維持してください。 + `AuthoritativeFreshRawBulkInsertTests` はraw scopeのactivationとsingle-owner transaction guard、正確な512 parameter / tail / write-count形状、32-entry bounded LRU、embedded NULを含むUnicode、stable date、NULL / 64-bit integer binding、provider専用routing除外、constraint時のrow-skip replay、deterministic interrupt rollback、hook failure cleanup、finalize count、直後のprovider再利用を固定します。fresh identity caseは正の64-bit file last-rowid、live tableと削除済みAUTOINCREMENT履歴の両方から得るreference-line floor、重複sequence rowのfail-closed、170/171 row境界、batch間insert、DML前のInt64枯渇、changed-row不一致、constraint / cancellation時のdiscard / reprepare、DML成功後だけのID公開、file単位SAVEPOINT rollbackを検証します。raw SQL-plan guardは`RETURNING`や相関input scanを含まない明示的な連続ID INSERTを要求し、provider、rebuild、incremental、MCP coverageは既存の`RETURNING`経路を維持します。source-symbol coverageは36-row reference batch境界と複数fileをまたぎ、fold済みname、fold済みdisplay name、legacy ASCII `NOCASE`、複数branchへの重複match、non-ASCII fallback拒否、nested equal-rank tieを検証し、null / empty containerを未解決のまま正確なsource IDを要求します。plan assertionはmaterializationで`idx_symbols_file`、lookupで3本すべてのTEMP partial indexを使い、どちらのsource scanも拒否します。savepoint failureとSQLite interruptはsnapshot rowとmain rowを一緒に復元しなければなりません。provider runtime testでは通常のfresh resolutionがTEMP tableを作成も参照もしないことも固定します。scope statsの`Completed=true`はnative statementのfinalizeとcancellation check成功後を示しますが、内部completion markerはreporting hookの正常return後にだけ設定するため、reporting-hook exceptionでは`Complete()`が失敗しscopeもnon-completedのままです。direct testをfresh / rebuild、update、claim-race fallback、MCP、mixed-language CLI、recoverable file failureのintegration assertionと対にし、authoritative empty CLI extraction区間だけがraw workを観測することを維持してください。 `PreparedCommandCacheTests.DbWriter_WithCache_FixedSavepointControlsReuseAcrossCommitRollbackAndTransactionRebind` は、depth 1 の SAVEPOINT / RELEASE / ROLLBACK が初回だけ cache miss となり、outer transaction が変わった後は cache hit となることを固定します。depth 2 以深を cache 外に保つ deep-savepoint test、control command を借りず outer scope を再利用可能なままにする cancelled-nested-begin test、同じ固定 statement 契約をすべての atomic marker surface へ横展開する metadata / FTS test と対にしてください。 - `PerformanceTests.AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations` は1 fileと10,000件ずつのchunk、symbol、issue、referenceについて、1回のwarm-up後にprovider / rawの3 measured pairを交互に実行します。productionの32 parameter provider境界、provider `RETURNING`、raw DONE / explicit-identity経路を使い、file timestamp textを含む永続化結果の完全一致をassertし、wall-clock ratio gateを設けずstage別 / totalの経過時間とcurrent-thread allocationを報告します。`CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter FullyQualifiedName~AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations`で実行してください。 + `PerformanceTests.AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations` は1 fileと10,000件ずつのchunk、symbol、issue、referenceについて、1回のwarm-up後にprovider / rawの3 measured pairを交互に実行します。productionの32 parameter provider境界、provider `RETURNING`、raw DONE / explicit-identity経路を使います。source fixtureはfile全体をspanするためreference-source IDもfile timestamp textとともにprovider / rawの永続化結果完全一致へ含めます。wall-clock ratio gateを設けずstage別 / totalの経過時間とcurrent-thread allocationを報告します。`CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter FullyQualifiedName~AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAllocations`で実行してください。 authoritative-fresh fold readiness の focused coverage は `DatabaseTests`、`IndexCommandRunnerTests`、`McpServerToolsCallTests` で分担します。built-in の empty-database CLI / MCP 経路が claim を一度だけ consume し、NULL column の検証を維持しつつ、保存 value の再 fold scan を省き、full validation と同じ readiness / version / fingerprint / language stamp、および Unicode、Markdown、C#、Nim、TypeScript の query result を生成することを固定します。初期状態で ownership table（`files`、`symbols`、`symbol_references`）のいずれかが非空の場合、owner が異なるか再利用された claim、`PRAGMA data_version` で観測される外部 commit、rebuild / update / legacy / public-writer 経路、custom plugin / pattern / post-extraction hook は fail closed であることも対にし、full validation が NULL と stale な非 NULL fold を引き続き拒否することを確認します。run barrier では custom producer を一度 active にしてから readiness 前に built-in-only へ reload し、最終 producer count が zero に戻っていても monotonic mutation generation の変化で full validation へ戻ることを固定します。状態不変の missing-directory と diagnostic-only publication では generation が変わらないことも確認します。 `BEGIN IMMEDIATE`成功直後のdeterministicなcancel testでは、raw transactionがrollbackされ、同じwriterが直後に別transactionを開始・commitできることを必須とします。 性能監査では、同一の repository-scale fresh fixture を交互に実行し、readiness finalization 区間を分離して、経過時間と `GC.GetAllocatedBytesForCurrentThread` を報告します。row 数に比例する managed allocation を取り除きつつ、row、stamp、query result が変わらないことを採用条件にします。wall-clock 計測は blocking CI assertion にせず、結果を記録したら一時 instrumentation を削除してください。 diff --git a/changelog.d/unreleased/+initial-full-index-reference-source-lookup.changed.md b/changelog.d/unreleased/+initial-full-index-reference-source-lookup.changed.md new file mode 100644 index 0000000000..8391e011bb --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-reference-source-lookup.changed.md @@ -0,0 +1,17 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.AuthoritativeFreshBulkInsert.cs + - src/CodeIndex/Database/DbWriter.AuthoritativeFreshReferenceSourceLookup.cs + - src/CodeIndex/Database/DbWriter.ReferenceSql.cs + - src/CodeIndex/Database/DbWriter.References.cs + - src/CodeIndex/Database/DbWriter.cs +--- + +## English + +- **Cold reference persistence probes one indexed per-file snapshot** — Empty-database raw loads now copy each relevant file's symbols once into a connection-local indexed TEMP table before resolving reference sources, preserving folded-name, display-name, legacy ASCII `NOCASE`, nesting, rollback, and query-result semantics while avoiding a persistent symbol lookup for every reference. + +## 日本語 + +- **初回 reference 永続化は indexed file snapshot を1回だけ探索** — 空DBへのraw loadは、reference sourceを解決する前に関連fileのsymbolをconnection-localなindexed TEMP tableへ各1回copyするようになりました。referenceごとのpersistent symbol lookupを避けながら、fold済みname、display name、legacy ASCII `NOCASE`、nesting、rollback、query resultの意味を維持します。 diff --git a/src/CodeIndex/Database/DbWriter.AuthoritativeFreshBulkInsert.cs b/src/CodeIndex/Database/DbWriter.AuthoritativeFreshBulkInsert.cs index df4b217ccf..f9d8ffbb27 100644 --- a/src/CodeIndex/Database/DbWriter.AuthoritativeFreshBulkInsert.cs +++ b/src/CodeIndex/Database/DbWriter.AuthoritativeFreshBulkInsert.cs @@ -114,6 +114,8 @@ internal static Func? DbContext.DropResourceListGenerationTriggersSql, _activeTransaction); cancellationToken.ThrowIfCancellationRequested(); + InitializeAuthoritativeFreshReferenceSourceLookup(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); var scope = new AuthoritativeFreshBulkInsertScope( this, @@ -332,10 +334,14 @@ internal void InsertReferences( var rows = end - start; using var interrupt = _writer.RegisterSqliteInterrupt(_cancellationToken); var sql = ReferenceInsertSqlCache.GetOrAdd( - (Rows: rows, FreshResolutionDefaults: true), + ( + Rows: rows, + FreshResolutionDefaults: true, + MaterializedFreshSourceLookup: true), static key => BuildReferenceInsertSql( key.Rows, - key.FreshResolutionDefaults)); + key.FreshResolutionDefaults, + key.MaterializedFreshSourceLookup)); var lease = RentStatementLease( AuthoritativeFreshRawInsertKind.References, rows, @@ -388,6 +394,15 @@ internal void InsertReferences( } } + internal void MaterializeReferenceSourceSymbols( + IReadOnlyList references) + { + EnsureCanExecute(); + _writer.MaterializeAuthoritativeFreshReferenceSourceLookup( + references, + _cancellationToken); + } + internal void Complete() { if (_disposed) diff --git a/src/CodeIndex/Database/DbWriter.AuthoritativeFreshReferenceSourceLookup.cs b/src/CodeIndex/Database/DbWriter.AuthoritativeFreshReferenceSourceLookup.cs new file mode 100644 index 0000000000..a7e5ec3e3f --- /dev/null +++ b/src/CodeIndex/Database/DbWriter.AuthoritativeFreshReferenceSourceLookup.cs @@ -0,0 +1,199 @@ +using CodeIndex.Models; +using Microsoft.Data.Sqlite; + +namespace CodeIndex.Database; + +public partial class DbWriter +{ + internal const string AuthoritativeFreshReferenceSourceSymbolsTableName = + "authoritative_fresh_reference_source_symbols"; + + private static readonly string InitializeAuthoritativeFreshReferenceSourceLookupSql = $""" + CREATE TEMP TABLE IF NOT EXISTS {AuthoritativeFreshReferenceSourceSymbolsTableName} ( + symbol_id INTEGER NOT NULL PRIMARY KEY, + file_id INTEGER NOT NULL, + name TEXT, + name_folded TEXT, + display_name_folded TEXT, + line INTEGER, + start_line INTEGER, + end_line INTEGER + ) WITHOUT ROWID; + + CREATE INDEX IF NOT EXISTS temp.idx_authoritative_fresh_source_name_folded + ON {AuthoritativeFreshReferenceSourceSymbolsTableName}(file_id, name_folded) + WHERE name_folded IS NOT NULL; + + CREATE INDEX IF NOT EXISTS temp.idx_authoritative_fresh_source_display_name_folded + ON {AuthoritativeFreshReferenceSourceSymbolsTableName}(file_id, display_name_folded) + WHERE display_name_folded IS NOT NULL; + + CREATE INDEX IF NOT EXISTS temp.idx_authoritative_fresh_source_name_nocase + ON {AuthoritativeFreshReferenceSourceSymbolsTableName}(file_id, name COLLATE NOCASE) + WHERE name_folded IS NULL; + + DELETE FROM temp.{AuthoritativeFreshReferenceSourceSymbolsTableName}; + """; + + private static readonly string ClearAuthoritativeFreshReferenceSourceLookupSql = $""" + DELETE FROM temp.{AuthoritativeFreshReferenceSourceSymbolsTableName} + """; + + private static readonly string PopulateAuthoritativeFreshReferenceSourceLookupSql = $""" + INSERT INTO temp.{AuthoritativeFreshReferenceSourceSymbolsTableName} ( + symbol_id, + file_id, + name, + name_folded, + display_name_folded, + line, + start_line, + end_line) + SELECT persisted.id, + persisted.file_id, + persisted.name, + persisted.name_folded, + persisted.display_name_folded, + persisted.line, + persisted.start_line, + persisted.end_line + FROM main.symbols AS persisted INDEXED BY idx_symbols_file + WHERE persisted.file_id = $file_id + """; + + internal static string PopulateAuthoritativeFreshReferenceSourceLookupSqlForTesting + => PopulateAuthoritativeFreshReferenceSourceLookupSql; + + private static string BuildMaterializedFreshReferenceSourceSymbolValueSql( + string referenceAlias) + => $""" + ( + SELECT candidate.symbol_id + FROM ( + SELECT source.symbol_id, + source.line, + source.start_line, + source.end_line + FROM temp.{AuthoritativeFreshReferenceSourceSymbolsTableName} AS source + WHERE {referenceAlias}.container_name IS NOT NULL + AND {referenceAlias}.container_name <> '' + AND source.file_id = {referenceAlias}.file_id + AND source.name_folded = {referenceAlias}.container_name_folded + + UNION + + SELECT source.symbol_id, + source.line, + source.start_line, + source.end_line + FROM temp.{AuthoritativeFreshReferenceSourceSymbolsTableName} AS source + WHERE {referenceAlias}.container_name IS NOT NULL + AND {referenceAlias}.container_name <> '' + AND source.file_id = {referenceAlias}.file_id + AND source.display_name_folded = {referenceAlias}.container_name_folded + + UNION + + SELECT source.symbol_id, + source.line, + source.start_line, + source.end_line + FROM temp.{AuthoritativeFreshReferenceSourceSymbolsTableName} AS source + WHERE {referenceAlias}.container_name IS NOT NULL + AND {referenceAlias}.container_name <> '' + AND source.file_id = {referenceAlias}.file_id + AND source.name_folded IS NULL + AND source.name = {referenceAlias}.container_name COLLATE NOCASE + ) AS candidate + WHERE {referenceAlias}.line BETWEEN COALESCE(candidate.start_line, candidate.line) + AND COALESCE(candidate.end_line, candidate.line) + ORDER BY (COALESCE(candidate.end_line, candidate.line) - + COALESCE(candidate.start_line, candidate.line)), + COALESCE(candidate.start_line, candidate.line) DESC, + candidate.symbol_id + LIMIT 1 + ) + """; + + internal static string BuildMaterializedFreshReferenceSourceSymbolValueSqlForTesting( + string referenceAlias) + => BuildMaterializedFreshReferenceSourceSymbolValueSql(referenceAlias); + + private void InitializeAuthoritativeFreshReferenceSourceLookup( + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var cancellationRegistration = RegisterSqliteInterrupt(cancellationToken); + try + { + Execute( + InitializeAuthoritativeFreshReferenceSourceLookupSql, + _activeTransaction); + } + catch (SqliteException exception) when ( + IsSqliteInterruptCancellation(exception, cancellationToken)) + { + throw new OperationCanceledException( + "Authoritative fresh source lookup initialization was interrupted.", + exception, + cancellationToken); + } + cancellationToken.ThrowIfCancellationRequested(); + } + + private void MaterializeAuthoritativeFreshReferenceSourceLookup( + IReadOnlyList references, + CancellationToken cancellationToken) + { + RequireCallerOwnedTransaction( + nameof(MaterializeAuthoritativeFreshReferenceSourceLookup)); + cancellationToken.ThrowIfCancellationRequested(); + + var fileIds = new List(); + var seenFileIds = new HashSet(); + foreach (var reference in references) + { + cancellationToken.ThrowIfCancellationRequested(); + if (reference.ContainerName is not { Length: > 0 } + || !seenFileIds.Add(reference.FileId)) + { + continue; + } + fileIds.Add(reference.FileId); + } + + using var cancellationRegistration = RegisterSqliteInterrupt(cancellationToken); + try + { + using (var clear = _conn.CreateCommand()) + { + clear.Transaction = _activeTransaction; + clear.CommandText = ClearAuthoritativeFreshReferenceSourceLookupSql; + clear.ExecuteNonQuery(); + } + + if (fileIds.Count > 0) + { + using var populate = _conn.CreateCommand(); + populate.Transaction = _activeTransaction; + populate.CommandText = PopulateAuthoritativeFreshReferenceSourceLookupSql; + var fileIdParameter = populate.Parameters.Add("$file_id", SqliteType.Integer); + foreach (var fileId in fileIds) + { + cancellationToken.ThrowIfCancellationRequested(); + fileIdParameter.Value = fileId; + populate.ExecuteNonQuery(); + } + } + } + catch (SqliteException exception) when ( + IsSqliteInterruptCancellation(exception, cancellationToken)) + { + throw new OperationCanceledException( + "Authoritative fresh source lookup materialization was interrupted.", + exception, + cancellationToken); + } + cancellationToken.ThrowIfCancellationRequested(); + } +} diff --git a/src/CodeIndex/Database/DbWriter.ReferenceSql.cs b/src/CodeIndex/Database/DbWriter.ReferenceSql.cs index 9e2a1ccbaf..33acf7ac2d 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceSql.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceSql.cs @@ -24,8 +24,16 @@ internal static Action? ReferenceInsertBindingWorkFo private static string BuildReferenceInsertSql( int rowCount, - bool useFreshReferenceResolutionDefaults) + bool useFreshReferenceResolutionDefaults, + bool useMaterializedFreshSourceLookup = false) { + if (useMaterializedFreshSourceLookup && !useFreshReferenceResolutionDefaults) + { + throw new ArgumentException( + "The materialized source lookup is only valid for authoritative fresh reference inserts.", + nameof(useMaterializedFreshSourceLookup)); + } + var sql = CreateBatchSqlBuilder(rowCount, estimatedCharsPerRow: 256); if (useFreshReferenceResolutionDefaults) { @@ -70,7 +78,9 @@ INSERT INTO symbol_references ( r.is_self_reference, r.is_mutual_recursion, r.target_qualifier, - {BuildReferenceSourceSymbolValueSql("r")}, + {(useMaterializedFreshSourceLookup + ? BuildMaterializedFreshReferenceSourceSymbolValueSql("r") + : BuildReferenceSourceSymbolValueSql("r"))}, 'unresolved', 0 FROM fresh_reference AS r @@ -101,8 +111,12 @@ INSERT INTO symbol_references ( internal static string BuildReferenceInsertSqlForTesting( int rowCount, - bool useFreshReferenceResolutionDefaults) - => BuildReferenceInsertSql(rowCount, useFreshReferenceResolutionDefaults); + bool useFreshReferenceResolutionDefaults, + bool useMaterializedFreshSourceLookup = false) + => BuildReferenceInsertSql( + rowCount, + useFreshReferenceResolutionDefaults, + useMaterializedFreshSourceLookup); private static void AppendReferenceInsertParameterTuple( StringBuilder sql, diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index a096e54202..2e404039bf 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -2179,6 +2179,10 @@ private void InsertReferencesCore( var useAuthoritativeFreshRawInsert = batchesAreAtomicInCaller && referenceLinesAreNew && _authoritativeFreshBulkInsertScope != null; + if (useAuthoritativeFreshRawInsert) + { + _authoritativeFreshBulkInsertScope!.MaterializeReferenceSourceSymbols(references); + } int rowsPerStatement = useAuthoritativeFreshRawInsert ? GetRowsPerAuthoritativeFreshRawInsertStatement( columnCount: ReferenceInsertParameterCountPerRow) @@ -2402,10 +2406,14 @@ private void InsertReferenceBatch( }; var cacheKey = ( Rows: rowsInBatch, - FreshResolutionDefaults: useFreshReferenceResolutionDefaults); + FreshResolutionDefaults: useFreshReferenceResolutionDefaults, + MaterializedFreshSourceLookup: false); var sql = ReferenceInsertSqlCache.GetOrAdd( cacheKey, - static key => BuildReferenceInsertSql(key.Rows, key.FreshResolutionDefaults)); + static key => BuildReferenceInsertSql( + key.Rows, + key.FreshResolutionDefaults, + key.MaterializedFreshSourceLookup)); var cmd = RentCommand(sql, c => AddReferenceInsertParameters(c, rowsInBatch)); try { diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 7ff674c2c6..45597b64a5 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -172,7 +172,9 @@ internal static Action? FreshBulkLoadPlannerStatistics private static readonly ConcurrentDictionary ChunkInsertSqlCache = new(); private static readonly ConcurrentDictionary SymbolInsertSqlCache = new(); private static readonly ConcurrentDictionary IssueInsertSqlCache = new(); - private static readonly ConcurrentDictionary<(int Rows, bool FreshResolutionDefaults), string> + private static readonly ConcurrentDictionary< + (int Rows, bool FreshResolutionDefaults, bool MaterializedFreshSourceLookup), + string> ReferenceInsertSqlCache = new(); private static readonly ConcurrentDictionary ReferenceLineUpsertSqlCache = new(); private static readonly ConcurrentDictionary ReferenceLineLookupSqlCache = new(); diff --git a/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs b/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs index df1857e82c..74dc28ee9a 100644 --- a/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs +++ b/tests/CodeIndex.Tests/AuthoritativeFreshRawBulkInsertTests.cs @@ -134,6 +134,13 @@ public void Scope_CoalescesResourceGenerationAndRestoresTriggersAcrossCommitAndR Assert.Equal( 0L, ScalarLong("SELECT COUNT(*) FROM files WHERE path = 'src/generation-rolled-back.cs'")); + Assert.Equal( + 0L, + ScalarLong($""" + SELECT COUNT(*) + FROM temp.sqlite_schema + WHERE name = '{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName}' + """)); using (var freshGraph = _writer.BeginReferenceGraphRefreshScope( forceFullRefresh: true, @@ -364,6 +371,473 @@ [new ReferenceRecord .ToArray(); } + [Fact] + public void ReferenceSourceLookup_PreservesMultiFileFoldFallbackAndNestedRankingAcrossBatches() + { + long firstFileId; + long secondFileId; + long firstNestedSourceId; + long aliasSourceId; + long duplicateProbeSourceId; + long legacyAsciiSourceId; + long secondNestedSourceId; + + using (var graph = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true)) + using (var transaction = _writer.BeginTransaction()) + using (var raw = _writer.BeginAuthoritativeFreshBulkInsertScope( + enabled: true, + CancellationToken.None)!) + { + firstFileId = InsertNewFile("src/source-lookup-a.cs"); + secondFileId = InsertNewFile("src/source-lookup-b.cs"); + _writer.InsertSymbols([ + SourceSymbol(firstFileId, "Caller", line: 1, startLine: 1, endLine: 100), + SourceSymbol(firstFileId, "Caller", line: 10, startLine: 10, endLine: 30), + SourceSymbol(firstFileId, "Caller", line: 10, startLine: 10, endLine: 30), + SourceSymbol( + firstFileId, + "Canonical", + line: 40, + startLine: 40, + endLine: 55, + displayNameFolded: "displayalias"), + SourceSymbol( + firstFileId, + "Dup", + line: 56, + startLine: 56, + endLine: 65, + displayNameFolded: "dup"), + SourceSymbol(firstFileId, "LegacyASCII", line: 66, startLine: 66, endLine: 75), + SourceSymbol(firstFileId, "ÅLegacy", line: 76, startLine: 76, endLine: 85), + SourceSymbol(secondFileId, "Caller", line: 1, startLine: 1, endLine: 100), + SourceSymbol(secondFileId, "Caller", line: 50, startLine: 50, endLine: 60), + ]); + Execute($""" + UPDATE symbols + SET name_folded = NULL + WHERE file_id = {firstFileId} + AND name IN ('LegacyASCII', 'ÅLegacy') + """); + + firstNestedSourceId = ScalarLong($""" + SELECT MIN(id) + FROM symbols + WHERE file_id = {firstFileId} + AND name = 'Caller' + AND start_line = 10 + """); + aliasSourceId = ScalarLong($""" + SELECT id FROM symbols + WHERE file_id = {firstFileId} AND name = 'Canonical' + """); + duplicateProbeSourceId = ScalarLong($""" + SELECT id FROM symbols + WHERE file_id = {firstFileId} AND name = 'Dup' + """); + legacyAsciiSourceId = ScalarLong($""" + SELECT id FROM symbols + WHERE file_id = {firstFileId} AND name = 'LegacyASCII' + """); + secondNestedSourceId = ScalarLong($""" + SELECT id FROM symbols + WHERE file_id = {secondFileId} + AND name = 'Caller' + AND start_line = 50 + """); + + var references = Enumerable.Range(0, 40) + .Select(index => SourceReference( + firstFileId, + $"nested_probe_{index}", + line: 15, + containerName: "Caller")) + .ToList(); + references.Add(SourceReference( + firstFileId, + "display_probe", + line: 45, + containerName: "DisplayAlias")); + references.Add(SourceReference( + firstFileId, + "duplicate_probe", + line: 60, + containerName: "Dup")); + references.Add(SourceReference( + firstFileId, + "legacy_ascii_probe", + line: 70, + containerName: "legacyascii")); + references.Add(SourceReference( + firstFileId, + "legacy_unicode_probe", + line: 80, + containerName: "ålegacy")); + references.Add(SourceReference( + firstFileId, + "null_container_probe", + line: 90, + containerName: null)); + references.Add(SourceReference( + firstFileId, + "empty_container_probe", + line: 91, + containerName: string.Empty)); + references.Add(SourceReference( + secondFileId, + "second_file_probe", + line: 55, + containerName: "Caller")); + + _writer.InsertReferencesForNewFilesInAtomicFileScope( + references, + refreshMutualRecursionFlags: false, + CancellationToken.None); + + Assert.Equal( + 2L, + ScalarLong($""" + SELECT COUNT(DISTINCT file_id) + FROM temp.{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName} + """)); + Assert.Equal( + 9L, + ScalarLong($""" + SELECT COUNT(*) + FROM temp.{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName} + """)); + Assert.Equal( + 3L, + ScalarLong(""" + SELECT COUNT(*) + FROM temp.sqlite_schema + WHERE type = 'index' + AND name LIKE 'idx_authoritative_fresh_source_%' + """)); + + raw.Complete(); + transaction.Commit(); + } + + Assert.Equal( + 40L, + ScalarLong($""" + SELECT COUNT(*) + FROM symbol_references + WHERE symbol_name LIKE 'nested_probe_%' + AND source_symbol_id = {firstNestedSourceId} + """)); + Assert.Equal(aliasSourceId, SourceSymbolId("display_probe")); + Assert.Equal(duplicateProbeSourceId, SourceSymbolId("duplicate_probe")); + Assert.Equal(legacyAsciiSourceId, SourceSymbolId("legacy_ascii_probe")); + Assert.Null(SourceSymbolId("legacy_unicode_probe")); + Assert.Null(SourceSymbolId("null_container_probe")); + Assert.Null(SourceSymbolId("empty_container_probe")); + Assert.Equal(secondNestedSourceId, SourceSymbolId("second_file_probe")); + + static SymbolRecord SourceSymbol( + long fileId, + string name, + int line, + int startLine, + int endLine, + string? displayNameFolded = null) + => new() + { + FileId = fileId, + Kind = "function", + Name = name, + Line = line, + StartLine = startLine, + EndLine = endLine, + DisplayNameFolded = displayNameFolded, + }; + + static ReferenceRecord SourceReference( + long fileId, + string symbolName, + int line, + string? containerName) + => new() + { + FileId = fileId, + SymbolName = symbolName, + ReferenceKind = "call", + Line = line, + Column = 1, + Context = $"{symbolName}();", + ContainerKind = containerName == null ? null : "function", + ContainerName = containerName, + }; + } + + [Fact] + public void ReferenceSourceLookup_QueryPlansUseRetainedAndPartialIndexesWithoutSourceScans() + { + using var graph = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true); + using var transaction = _writer.BeginTransaction(); + using var raw = _writer.BeginAuthoritativeFreshBulkInsertScope( + enabled: true, + CancellationToken.None)!; + + var materializationPlan = ExplainQueryPlan( + DbWriter.PopulateAuthoritativeFreshReferenceSourceLookupSqlForTesting, + command => command.Parameters.AddWithValue("$file_id", 1L)); + Assert.Contains( + materializationPlan, + detail => detail.Contains("idx_symbols_file", StringComparison.Ordinal)); + Assert.DoesNotContain( + materializationPlan, + detail => detail.Contains("SCAN persisted", StringComparison.OrdinalIgnoreCase)); + + var sourceValueSql = + DbWriter.BuildMaterializedFreshReferenceSourceSymbolValueSqlForTesting("r"); + var sourceLookupPlan = ExplainQueryPlan($""" + WITH reference_row(file_id, line, container_name, container_name_folded) AS ( + VALUES (1, 15, 'Caller', 'caller') + ) + SELECT {sourceValueSql} + FROM reference_row AS r + """); + Assert.Contains( + sourceLookupPlan, + detail => detail.Contains( + "idx_authoritative_fresh_source_name_folded", + StringComparison.Ordinal)); + Assert.Contains( + sourceLookupPlan, + detail => detail.Contains( + "idx_authoritative_fresh_source_display_name_folded", + StringComparison.Ordinal)); + Assert.Contains( + sourceLookupPlan, + detail => detail.Contains( + "idx_authoritative_fresh_source_name_nocase", + StringComparison.Ordinal)); + Assert.DoesNotContain( + sourceLookupPlan, + detail => detail.Contains("SCAN source", StringComparison.OrdinalIgnoreCase)); + + raw.Complete(); + transaction.Commit(); + + IReadOnlyList ExplainQueryPlan( + string sql, + Action? bind = null) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = "EXPLAIN QUERY PLAN " + sql; + bind?.Invoke(command); + using var reader = command.ExecuteReader(); + var plan = new List(); + while (reader.Read()) + plan.Add(reader.GetString(3)); + return plan; + } + } + + [Fact] + public void ReferenceSourceLookup_FileSavepointRollbackRestoresPreviousRowsAndReprepares() + { + long firstFileId; + long thirdFileId; + using (var graph = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true)) + using (var outerTransaction = _writer.BeginTransaction()) + using (var raw = _writer.BeginAuthoritativeFreshBulkInsertScope( + enabled: true, + CancellationToken.None)!) + { + using (var firstFile = _writer.BeginTransaction()) + { + firstFileId = InsertNewFile("src/source-savepoint-a.cs"); + _writer.InsertSymbols([ + CreateSourceSymbol(firstFileId, "CallerA"), + ]); + _writer.InsertReferencesForNewFilesInAtomicFileScope( + [CreateSourceReference(firstFileId, "first_probe", "CallerA")], + refreshMutualRecursionFlags: false, + CancellationToken.None); + firstFile.Commit(); + } + + Assert.Equal( + 1L, + ScalarLong($""" + SELECT COUNT(*) + FROM temp.{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName} + WHERE file_id = {firstFileId} + """)); + + using (var failedFile = _writer.BeginTransaction()) + { + var failedFileId = InsertNewFile("src/source-savepoint-failed.cs"); + _writer.InsertSymbols([ + CreateSourceSymbol(failedFileId, "FailedCaller"), + ]); + Execute($""" + CREATE TEMP TRIGGER reject_materialized_source_reference + BEFORE INSERT ON symbol_references + WHEN NEW.file_id = {failedFileId} + BEGIN + SELECT RAISE(ABORT, 'reject materialized source reference'); + END; + """); + Assert.Throws(() => + _writer.InsertReferencesForNewFilesInAtomicFileScope( + [CreateSourceReference( + failedFileId, + "failed_probe", + "FailedCaller")], + refreshMutualRecursionFlags: false, + CancellationToken.None)); + } + + Assert.Equal( + 0L, + ScalarLong(""" + SELECT COUNT(*) FROM files + WHERE path = 'src/source-savepoint-failed.cs' + """)); + Assert.Equal( + 1L, + ScalarLong($""" + SELECT COUNT(*) + FROM temp.{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName} + WHERE file_id = {firstFileId} + """)); + + using (var thirdFile = _writer.BeginTransaction()) + { + thirdFileId = InsertNewFile("src/source-savepoint-c.cs"); + _writer.InsertSymbols([ + CreateSourceSymbol(thirdFileId, "CallerC"), + ]); + _writer.InsertReferencesForNewFilesInAtomicFileScope( + [CreateSourceReference(thirdFileId, "third_probe", "CallerC")], + refreshMutualRecursionFlags: false, + CancellationToken.None); + thirdFile.Commit(); + } + + raw.Complete(); + outerTransaction.Commit(); + } + + Assert.Equal(firstFileId, SourceFileId("first_probe")); + Assert.Equal(thirdFileId, SourceFileId("third_probe")); + Assert.Equal( + 0L, + ScalarLong(""" + SELECT COUNT(*) FROM symbol_references + WHERE symbol_name = 'failed_probe' + """)); + + static SymbolRecord CreateSourceSymbol(long fileId, string name) + => new() + { + FileId = fileId, + Kind = "function", + Name = name, + Line = 1, + StartLine = 1, + EndLine = 10, + }; + + static ReferenceRecord CreateSourceReference( + long fileId, + string symbolName, + string containerName) + => new() + { + FileId = fileId, + SymbolName = symbolName, + ReferenceKind = "call", + Line = 5, + Column = 1, + Context = $"{symbolName}();", + ContainerKind = "function", + ContainerName = containerName, + }; + } + + [Fact] + public void ReferenceSourceLookup_InterruptRollsBackTempAndMainState() + { + using var cancellation = new CancellationTokenSource(); + _db.Connection.CreateFunction( + "cancel_authoritative_fresh_source_lookup", + () => + { + cancellation.Cancel(); + return 0; + }); + + OperationCanceledException exception; + using (var graph = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true)) + using (var transaction = _writer.BeginTransaction()) + using (var raw = _writer.BeginAuthoritativeFreshBulkInsertScope( + enabled: true, + cancellation.Token)!) + { + var fileId = InsertNewFile("src/source-lookup-interrupted.cs"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "Caller", + Line = 1, + StartLine = 1, + EndLine = 10, + }, + ]); + Execute($""" + CREATE TEMP TRIGGER cancel_authoritative_fresh_source_materialization + BEFORE INSERT ON {DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName} + BEGIN + SELECT cancel_authoritative_fresh_source_lookup(); + END; + """); + exception = Assert.Throws(() => + _writer.InsertReferencesForNewFilesInAtomicFileScope( + [new ReferenceRecord + { + FileId = fileId, + SymbolName = "interrupted_probe", + ReferenceKind = "call", + Line = 5, + Column = 1, + Context = "interrupted_probe();", + ContainerKind = "function", + ContainerName = "Caller", + }], + refreshMutualRecursionFlags: false, + cancellation.Token)); + } + + Assert.Equal(cancellation.Token, exception.CancellationToken); + var sqliteException = Assert.IsType(exception.InnerException); + Assert.Equal(9, sqliteException.SqliteErrorCode); + Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM files")); + Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM symbols")); + Assert.Equal(0L, ScalarLong("SELECT COUNT(*) FROM symbol_references")); + Assert.Equal( + 0L, + ScalarLong($""" + SELECT COUNT(*) + FROM temp.sqlite_schema + WHERE name = '{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName}' + """)); + } + [Fact] public void StatementCache_EvictsLeastRecentlyUsedAndFinalizesEveryShape() { @@ -1160,6 +1634,37 @@ FROM reference_lines return Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture); } + private long? SourceSymbolId(string symbolName) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = """ + SELECT source_symbol_id + FROM symbol_references + WHERE symbol_name = @symbol_name + """; + command.Parameters.AddWithValue("@symbol_name", symbolName); + var value = command.ExecuteScalar(); + return value == null || value == DBNull.Value + ? null + : Convert.ToInt64(value, CultureInfo.InvariantCulture); + } + + private long? SourceFileId(string symbolName) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = """ + SELECT source.file_id + FROM symbol_references AS reference + LEFT JOIN symbols AS source ON source.id = reference.source_symbol_id + WHERE reference.symbol_name = @symbol_name + """; + command.Parameters.AddWithValue("@symbol_name", symbolName); + var value = command.ExecuteScalar(); + return value == null || value == DBNull.Value + ? null + : Convert.ToInt64(value, CultureInfo.InvariantCulture); + } + private long ResourceListGeneration() => ScalarLong(""" SELECT CAST(value AS INTEGER) diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 4ee6d11863..ec19494085 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -9637,6 +9637,12 @@ public void BatchNumericParameterSql_IsOneOriginForTwentyFiveColumnSymbolsAndRef rowCount: 2, useFreshReferenceResolutionDefaults: true), expectedParameterCount: 28); + AssertNumericParameterOrdinals( + DbWriter.BuildReferenceInsertSqlForTesting( + rowCount: 2, + useFreshReferenceResolutionDefaults: true, + useMaterializedFreshSourceLookup: true), + expectedParameterCount: 28); static void AssertNumericParameterOrdinals(string sql, int expectedParameterCount) { diff --git a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs index aa6cf444b7..fef6946785 100644 --- a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs +++ b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs @@ -76,6 +76,13 @@ public void InsertReferences_FreshDefaultsKeepParameterShapeAndUseSeparateCached { var fileId = InsertFile("src/provisional.py", "python"); _writer.InsertSymbols([CreateSymbol(fileId, "Caller", line: 1)]); + Assert.Equal( + 0L, + ScalarLong($""" + SELECT COUNT(*) + FROM temp.sqlite_schema + WHERE name = '{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName}' + """)); var observedWork = new List(); var previousHook = DbWriter.ReferenceInsertBindingWorkForTesting; try @@ -137,6 +144,13 @@ FROM symbol_references WHERE symbol_name = 'Fresh' AND source_symbol_id IS NOT NULL """)); + Assert.Equal( + 0L, + ScalarLong($""" + SELECT COUNT(*) + FROM temp.sqlite_schema + WHERE name = '{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName}' + """)); Assert.Equal( 1, ScalarLong(""" @@ -149,6 +163,10 @@ AND source_symbol_id IS NULL var freshSql = DbWriter.BuildReferenceInsertSqlForTesting( rowCount: 2, useFreshReferenceResolutionDefaults: true); + var materializedFreshSql = DbWriter.BuildReferenceInsertSqlForTesting( + rowCount: 2, + useFreshReferenceResolutionDefaults: true, + useMaterializedFreshSourceLookup: true); var standardSql = DbWriter.BuildReferenceInsertSqlForTesting( rowCount: 2, useFreshReferenceResolutionDefaults: false); @@ -160,10 +178,33 @@ AND source_symbol_id IS NULL Assert.Contains("ORDER BY (COALESCE(s.end_line", freshSql, StringComparison.Ordinal); Assert.Equal(28, CountOccurrences(freshSql, "?")); Assert.DoesNotContain("?0", freshSql, StringComparison.Ordinal); + Assert.Contains("FROM symbols AS s", freshSql, StringComparison.Ordinal); + Assert.DoesNotContain( + DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName, + freshSql, + StringComparison.Ordinal); + Assert.Contains( + $"FROM temp.{DbWriter.AuthoritativeFreshReferenceSourceSymbolsTableName} AS source", + materializedFreshSql, + StringComparison.Ordinal); + Assert.DoesNotContain("FROM symbols AS s", materializedFreshSql, StringComparison.Ordinal); + Assert.Equal(2, CountOccurrences(materializedFreshSql, "UNION")); + Assert.DoesNotContain("UNION ALL", materializedFreshSql, StringComparison.Ordinal); + Assert.Contains( + "COALESCE(candidate.start_line, candidate.line) DESC", + materializedFreshSql, + StringComparison.Ordinal); + Assert.Equal(28, CountOccurrences(materializedFreshSql, "?")); + Assert.DoesNotContain("?0", materializedFreshSql, StringComparison.Ordinal); Assert.DoesNotContain("WITH fresh_reference(", standardSql, StringComparison.Ordinal); Assert.DoesNotContain("source_symbol_id", standardSql, StringComparison.Ordinal); Assert.Equal(28, CountOccurrences(standardSql, "?")); Assert.DoesNotContain("?0", standardSql, StringComparison.Ordinal); + Assert.Throws(() => + DbWriter.BuildReferenceInsertSqlForTesting( + rowCount: 1, + useFreshReferenceResolutionDefaults: false, + useMaterializedFreshSourceLookup: true)); var freshRefresh = DbWriter.SelectReferenceSourceRefreshSqlForTesting( useFreshReferenceResolutionDefaults: true, diff --git a/tests/CodeIndex.Tests/PerformanceTests.cs b/tests/CodeIndex.Tests/PerformanceTests.cs index d0e630823b..d046a805ed 100644 --- a/tests/CodeIndex.Tests/PerformanceTests.cs +++ b/tests/CodeIndex.Tests/PerformanceTests.cs @@ -105,10 +105,10 @@ public void AuthoritativeFreshRawBulkInsert_ReportsProviderParityElapsedAndAlloc { FileId = 1, Kind = "function", - Name = $"target_{index}", + Name = index == 0 ? "caller" : $"target_{index}", Line = index + 1, - StartLine = index + 1, - EndLine = index + 1, + StartLine = index == 0 ? 1 : index + 1, + EndLine = index == 0 ? RowCount : index + 1, Signature = index % 2 == 0 ? null : $"void target_{index}()", }) .ToArray(); @@ -249,7 +249,8 @@ private static RawBulkInsertBenchmarkSample RunRawBulkInsertBenchmark( (SELECT generated FROM files WHERE id = 1), (SELECT hex(CAST(content AS BLOB)) FROM chunks WHERE chunk_index = 0), (SELECT COUNT(*) FROM symbols WHERE signature IS NULL), - (SELECT COUNT(*) FROM symbol_references WHERE context IS NULL) + (SELECT COUNT(*) FROM symbol_references WHERE context IS NULL), + (SELECT COUNT(*) FROM symbol_references WHERE source_symbol_id IS NOT NULL) """; using var reader = snapshotCommand.ExecuteReader(); Assert.True(reader.Read()); From b9552be13f03f107d1ac3e96ca11f2d0c98a3836 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 30 Aug 2026 01:22:00 +0900 Subject: [PATCH 10/11] Bound reference resolution to candidate symbols --- DEVELOPER_GUIDE.md | 98 +++++---- TESTING_GUIDE.md | 16 +- ...full-index-candidate-resolution.changed.md | 17 ++ .../DbWriter.ReferenceGraphRefreshScope.cs | 24 ++- src/CodeIndex/Database/DbWriter.References.cs | 138 ++++++++---- src/CodeIndex/Database/DbWriter.cs | 7 + .../ReferenceSecondaryIndexBulkLoadGuard.cs | 30 ++- .../Database/ReferenceSecondaryIndexSql.cs | 11 +- tests/CodeIndex.Tests/DatabaseTests.cs | 202 +++++++++++++++++- .../FreshReferenceResolutionTests.cs | 136 +++++++++++- ...ommandRunnerReferenceIndexBulkLoadTests.cs | 37 ++-- .../McpServerToolsCallTests.cs | 10 +- ...ferenceSecondaryIndexBulkLoadGuardTests.cs | 82 ++++++- 13 files changed, 691 insertions(+), 117 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-candidate-resolution.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index bbf239c040..5cc8308393 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -398,9 +398,14 @@ indexes on `symbol_references` until raw reference persistence completes. The reverse candidate-symbol lookup remains available during raw persistence and is dropped only when an actual graph refresh is about to delete or materialize candidate rows, so marker-only and high-cardinality no-op updates do not rebuild -that whole index. The candidate primary key remains available for reference-scoped -materialization and resolution, and the file and reference-line maintenance indexes -normally remain available during the load. The sole exception is an authoritative +that whole index. Candidate construction uses the `(reference_id, symbol_id)` +primary key without maintaining the reverse B-tree. Immediately after candidate +construction, restore `idx_symbol_ref_candidates_symbol` before renting or +preparing the separate resolution command, so target-fact materialization can use +bounded `(symbol_id, reference_id)` existence seeks. The candidate primary key +remains available for reference-scoped materialization and resolution, and the file +and reference-line maintenance indexes normally remain available during the load. +The sole exception is an authoritative empty-database first CLI full scan whose transaction-local recheck still owns the fresh-resolution claim: while it persists raw references, it also defers `idx_symbol_refs_reference_line` and `idx_reference_lines_file_line`. It restores @@ -423,14 +428,12 @@ expression single-evaluation: repeating it in both `SET` and `WHERE` causes fresh large graphs to perform the same random B-tree probes twice. When a TypeScript augmentation rebuild owns the sole graph pass, restore every ordinary graph/query index before readiness, then drop the reverse candidate-symbol -lookup immediately before augmentation candidate population and keep it deferred -through that graph pass. Transactional full -scans restore that final index after readiness work and before committing the outer -full-scan transaction, preserving atomic schema rollback on cancellation or -failure. Recoverable scoped updates and MCP indexing retain guard ownership through -the readiness transaction commit and restore the final index immediately afterward. -Both lifecycles keep readiness queries available without maintaining the candidate -B-tree row by row. MCP uses the recoverable lifecycle whenever its established +lookup immediately before augmentation candidate population. Restore that lookup +immediately after population and before preparing resolution. Transactional rollback +and recoverable disposal still repair failures before that boundary; successful +resolution and readiness observe the canonical candidate lookup without maintaining +the reverse B-tree row by row during candidate inserts. MCP uses the recoverable +lifecycle whenever its established dirty-byte policy selects FTS bulk loading, and restores every index on completion or disposal. Schema initialization and read repair must use the same canonical index catalog so every path converges on an identical schema. @@ -462,8 +465,10 @@ non-mutating skips, unchanged targets, and sparsely mutating target sets keep ev Any preflight uncertainty keeps staging enabled. This preflight is only a cost decision—the file loop must repeat its live authoritative lookup so changes after the snapshot are still indexed with all indexes present. -Keep the query-only set deferred through identity and resolution work, -restore the three reverse-edge indexes immediately before mutual recursion, then +Keep the query-only set deferred through identity and resolution work, except that +the candidate-symbol reverse lookup returns between candidate population and +resolution-command preparation. Restore the three reverse-edge indexes immediately +before mutual recursion, then restore the remainder after that update. Small scoped updates must keep every index in place so a fixed rebuild cost does not dominate the update. @@ -504,13 +509,20 @@ family once. Only the final projection expands a matched family back to every physical member. This preserves row-per-symbol candidates for partial types while avoiding repeated compatibility and ambiguity work for every partial declaration. -Resolution also materializes the nullable target-family key once per target symbol -into a primary-keyed TEMP fact table. Full, fresh, differential, and retained -refreshes populate all symbols; scoped refreshes first deduplicate target symbol -IDs reachable from dirty-reference candidates. Resolution must join candidates to -that fact by symbol ID instead of rebuilding the language/path/container/name key -for every physical candidate. Preserve a `NULL` key when legacy target language is -missing, while still resolving a single valid candidate by ID. +Resolution also materializes the nullable target-family key once per candidate-bearing +target symbol into a primary-keyed TEMP fact table. Full, fresh, differential, and +retained refreshes filter that population with indexed candidate-symbol existence +probes; scoped refreshes first deduplicate target symbol IDs reachable from +dirty-reference candidates. Candidate construction and resolution stay in separate +SQLite commands, and the resolution command is rented and prepared only after the +reverse lookup is restored. Resolution joins candidates to facts by symbol ID instead +of rebuilding the language/path/container/name key for every physical candidate. +Singleton-family detection preserves `COUNT(DISTINCT)` NULL semantics without a +per-group distinct set: at least one non-NULL key must exist and the BINARY minimum +must be null-safely `IS` the BINARY maximum. All-NULL groups remain non-families, +NULL plus one non-NULL family retains that family, duplicate keys collapse, and +binary-distinct keys remain ambiguous. A single physical legacy candidate with a +NULL key still resolves by ID. Repository-wide incremental scans load stat-reuse candidates with one SQLite statement before the C# contract prepass and parallel extraction. Each candidate @@ -1810,7 +1822,8 @@ the compatibility filter are treated as non-authoritative until a normal index r their candidates. Existing-index, rebuild, and retained-graph finalization paths compute candidate count, minimum -symbol ID, distinct target-family count, and stable target key in one correlated aggregate per +symbol ID, a single-target-family flag from a positive non-NULL count plus BINARY +`MIN(...) IS MAX(...)`, and the stable target key in one correlated aggregate per reference. Keep these four resolution fields on the row-value assignment path; separate scalar subqueries multiply the candidate-index and symbol/file lookup work on large graphs. The language/name families that @@ -4647,8 +4660,12 @@ fresh な CLI scan と明示的 rebuild は、raw reference の永続化が完 `symbol_references` の query / graph 用 secondary index を遅延します。candidate-symbol の reverse lookup は raw persistence 中は維持し、実際の graph refresh が candidate row を削除・ 構築する直前だけ外すため、marker-only / 高 cardinality no-op update はこの index 全体を -再構築しません。reference scope の materialization / resolution に使う candidate primary key は -維持します。load 中も通常は file と reference-line の保守用 index を残します。唯一の例外は、 +再構築しません。candidate 構築は `(reference_id, symbol_id)` primary key を使い、reverse B-tree +を行ごとに保守しません。candidate 構築完了直後かつ独立した resolution command の rent / prepare +前に `idx_symbol_ref_candidates_symbol` を復元し、target fact の materialization は bounded な +`(symbol_id, reference_id)` existence seek を使います。reference scope の materialization / +resolution に使う candidate primary key は維持します。load 中も通常は file と reference-line の +保守用 index を残します。唯一の例外は、 transaction-local な再確認後も fresh-resolution claim を所有する authoritative な空DB初回CLI full scan です。この経路だけは raw reference の永続化中に `idx_symbol_refs_reference_line` と `idx_reference_lines_file_line` も遅延し、candidate、 @@ -4663,12 +4680,11 @@ identity / resolution finalization 中は query index を遅延したままに legacy NOCASE、resolved reverse-edge の3本だけを復元し、残りの query index は mutual update 後に戻します。TypeScript augmentation rebuild が唯一の graph pass を担当する場合は、readiness 前に通常の graph / query index を復元し、augmentation の candidate 構築直前にだけ -candidate-symbol reverse lookup を外して graph pass の完了まで遅延します。transactional full scan は readiness work 後 -かつ outer full-scan transaction の commit 前に最後の1本を復元し、cancellation や失敗時の -schema rollback を原子的に保ちます。recoverable scoped update と MCP indexing は readiness -transaction の commit まで guard ownership を保持し、その直後に最後の index を復元します。 -どちらの lifecycle も readiness query を利用可能なまま candidate B-tree の行ごとの保守を -省きます。MCP は既定の dirty-byte policy が FTS bulk load を選ぶ場合に recoverable lifecycle +candidate-symbol reverse lookup を外します。candidate 構築直後かつ resolution の prepare 前に +この lookup を復元します。その境界より前の失敗は transactional rollback / recoverable disposal +が修復し、正常な resolution と readiness は canonical な candidate lookup を利用できます。 +candidate insert 中だけ reverse B-tree の行ごとの保守を省きます。MCP は既定の dirty-byte policy +が FTS bulk load を選ぶ場合に recoverable lifecycle を使い、正常完了時と dispose 時の両方で全 index を復元します。schema initialization と read repair は同じ canonical index catalog を使い、すべての経路が同一の最終 schema に収束する状態を 保ってください。 @@ -4695,8 +4711,9 @@ index 退避を使います。scoped update には workspace 全体の authorita unchanged、または sparse mutation の target 集合では全 index を維持します。preflight に不確実性があれば保守的に staging を維持します。この preflight は cost 判定に限り、snapshot 後の 変更も全 index を維持したまま更新できるよう、file loop は authoritative な live lookup を必ず再実行 -してください。identity / resolution 中は query-only 集合を遅延したままにし、mutual recursion の -直前に reverse-edge 用3本を復元して、その update 後に残りを戻してください。小規模 scoped +してください。identity / resolution 中は query-only 集合を遅延したままにしますが、candidate-symbol +reverse lookup だけは candidate 構築後かつ resolution command の prepare 前に復元します。mutual +recursion の直前に reverse-edge 用3本を復元して、その update 後に残りを戻してください。小規模 scoped update は固定的な再構築 cost が更新時間を支配しないよう、全 index を維持します。 full mutual-recursion update は、call-like または非canonicalな row ごとに望ましい flag を 1回 materialize してから変更を適用します。相関 reverse-edge 式を `SET` と `WHERE` の @@ -4734,12 +4751,18 @@ family単位で1回だけ行います。一致したfamilyを全物理memberへ これによりpartial typeのsymbolごとのcandidate行を維持しつつ、各partial宣言でcompatibilityとambiguity 判定を繰り返しません。 -resolution は nullable な target-family key も target symbol ごとに1回だけ primary-keyed TEMP -fact table へ materialize します。full / fresh / differential / retained refresh は全 symbol を投入し、 -scoped refresh は dirty-reference candidate から到達する target symbol ID を先に重複排除します。 +resolution は nullable な target-family key も candidate を持つ target symbol ごとに1回だけ +primary-keyed TEMP fact table へ materialize します。full / fresh / differential / retained refresh は +indexed な candidate-symbol existence probe で対象を限定し、scoped refresh は dirty-reference +candidate から到達する target symbol ID を先に重複排除します。candidate 構築と resolution は別の +SQLite command とし、reverse lookup の復元後にだけ resolution command を rent / prepare します。 resolution は物理 candidate ごとに language / path / container / name key を再構築せず、symbol IDで -このfactへjoinしてください。legacy targetのlanguageが欠ける場合はkeyを`NULL`のまま保ちつつ、 -有効candidateが1件ならIDによるresolved状態を維持します。 +このfactへjoinしてください。singleton family 判定は per-group の DISTINCT set を作らず、非NULL +key が1件以上あり、BINARY の `MIN(...) IS MAX(...)` であることを使って従来の +`COUNT(DISTINCT)` NULL semantics を維持します。all-NULL は family なし、NULL と1つの非NULL +family はその family、重複 key は1 family、BINARY で異なる key は ambiguous です。legacy target +の language が欠ける場合は key を `NULL` のまま保ちつつ、物理 candidate が1件なら ID による +resolved 状態を維持します。 リポジトリ全体の incremental scan は、C# contract prepass と parallel extraction の前に stat-reuse 候補を 1 回の SQLite statement で読みます。各候補は引き続き最新の filesystem @@ -5972,7 +5995,8 @@ Java の reference resolution は変更しません。 作成された index は、通常の index 更新で candidate を再構築するまで非 authoritative として扱います。 既存index、rebuild、retained graph の reference finalization は、candidate count、最小 symbol ID、 -distinct target-family count、安定 target key を reference ごとに1回の correlated aggregate で +非NULL count が正で BINARY の `MIN(...) IS MAX(...)` となる single-target-family flag、安定 target key +を reference ごとに1回の correlated aggregate で 計算します。この4つの resolution field は row-value assignment のまま維持してください。 scalar subquery を分けると、大規模 graph で candidate index と symbol/file lookup が重複します。 global に一意な language/name family は diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 7c0fbc3550..cfd3a7424a 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -213,7 +213,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `DbReaderCSharpUsingCatalogTests` keeps reader-scoped C# using-catalog behavior explicit: file-scoped namespaces extend through EOF, nested namespace imports stay within the innermost inclusive scope, repeated active path/line lookups reuse the same cached set, and local/global alias shadowing preserves resolvable chains while terminating cycles. `CSharpBaseListParserTests` keeps the SQLite-free base-list grammar matrix together, including generic constraints, nested generic/tuple/array syntax, alias qualifiers, primary/base constructors, declaration terminators, and the distinct reader type-reference and metadata head-identifier projections. Maintenance-lookup coverage keeps the `files(checksum)`, `files(path COLLATE NOCASE)`, and `file_issues(file_id, kind)` indexes aligned with their predicates and requires `EXPLAIN QUERY PLAN` index `SEARCH` operations for checksum purge, ASCII case-alias lookup, reusable-stat issue probes, and directory/stem rename candidates. The case-alias fixture uses changed content so checksum cannot hide a missing path lookup, and managed validation remains authoritative because SQLite `NOCASE` is only an ASCII prefilter, not a Unicode casing contract. Keep wildcard-bearing, extensionless, and near-match stem semantics in one fixture. Scoped-cleanup planning coverage unions checksum and exact same-directory/stem candidates into ascending deduplicated ID snapshots, merges overlapping plans, proves apply does not absorb matching rows added after planning, and rereads planned IDs immediately before apply so a deleted C# contract that reappears is deferred until a clean retry. The grouped C# fixture gives many targets one common checksum/stem and requires candidate-reader work to grow with unique keys plus returned rows rather than target-count squared; it also asserts that C# pre-workspace planning never deletes matching non-C# rows. - Reference-resolution lookup coverage keeps the exact fresh-schema and read-migration index sets aligned for file/name, retained folded/NOCASE composite prefixes, resolved reverse edges, and the partial unresolved-mutual path. Assert the absence of all six retired single-prefix indexes plus the former all-row mutual index after migration and reopen, exact column order/collation/partial predicates, and `EXPLAIN QUERY PLAN` `SEARCH` operations for both full and scoped mutual SQL. Preserve fold-ready and legacy-fallback CLI/MCP exact-query signals, and rerun the same-name/different-container plus partial-class-over-same-file ranking fixtures when these indexes change. + Reference-resolution lookup coverage keeps the exact fresh-schema and read-migration index sets aligned for file/name, retained folded/NOCASE composite prefixes, resolved reverse edges, and the partial unresolved-mutual path. Assert the absence of all six retired single-prefix indexes plus the former all-row mutual index after migration and reopen, exact column order/collation/partial predicates, and `EXPLAIN QUERY PLAN` `SEARCH` operations for both full and scoped mutual SQL. Full, fresh, differential, and retained target-fact SQL must use `idx_symbol_ref_candidates_symbol` for bounded candidate-existence seeks and exclude unrelated symbols from TEMP facts; scoped SQL must retain its dirty-candidate primary-key seeks. Assert that resolution SQL has no `COUNT(DISTINCT)` set, uses BINARY `MIN(...) IS MAX(...)`, and matches the old aggregate for empty, all-NULL, NULL-plus-one-family, duplicate-family, binary case-variant, distinct-family, and single physical NULL-key inputs. Preserve fold-ready and legacy-fallback CLI/MCP exact-query signals, and rerun the same-name/different-container plus partial-class-over-same-file ranking fixtures when these indexes change. Repository-metadata graph coverage indexes TOML local paths and application-manifest assembly dependencies through the normal writer, then requires explicit `project_reference` / `dependency` caller queries so capability advertising cannot drift from persisted graph behavior. Reference-identity write coverage uses per-column SQLite audit triggers around one self-resolving reference. It requires a stable rerun to write no source, resolution, self, or mutual rows; corrupts all four phases; aborts target resolution to prove the earlier source update rolls back; then requires exactly one repair write per phase and another stable zero-write rerun. Reusable-stat snapshot semantics keep one valid row beside two rows from the same stale extractor language plus NULL, text, integer, and invalid-timestamp stat storage, so malformed-row filtering and cached stale-language rejection remain one contract. @@ -976,9 +976,9 @@ Use the inventory below before adding or moving a test class: - SQLite sensitive-fixture initialization, disposal, and boundary clearing should share one replaced callback scope; the boundary assertion can follow lifecycle assertions without rebuilding global hook state. - SQLite connection policy builder coverage should verify connection strings, command timeout, and status diagnostics in one test; these allocation-only checks do not need three runner cases. - SQLite command parameter builders should verify primitive types, stable dates, and copied parameter shapes on one command fixture rather than allocating three independent test cases. -- Reference-secondary-index bulk-load coverage should assert the exact canonical schema at raw persistence, candidate deferral, identity start, graph-required restore, mutual-recursion start, and final query restore. Raw persistence keeps the reverse candidate-symbol lookup while ordinary query/graph indexes are absent; an actual graph refresh drops it immediately before candidate deletion/materialization. The ordinary middle graph set is exactly unresolved-folded, legacy NOCASE, and resolved reverse-edge, and its three plans plus candidate reference-primary-key lookups must remain usable. When TypeScript augmentation owns the graph pass, assert that all ordinary graph/query indexes return before readiness, the reverse candidate-symbol lookup remains available through marker/grouping work, and it is absent only from candidate population through mutual recursion. For transactional full scans, assert that it returns after readiness work but before the outer full-scan transaction commits; for recoverable scoped updates and MCP indexing, assert that guard ownership remains live through the readiness transaction commit and that the index returns immediately afterward. Also prove direct `changes()` preservation, no-graph completion without candidate-index DDL, graph-stage cancellation/disposal recovery, abandoned-stage schema repair, and transactional rollback without relying on initialization repair. A guard still removes legacy single-prefix/all-row-mutual indexes without restoring them recoverably. -- Fresh bulk-load planner-statistics coverage should seed only the minimum file, symbol, and reference rows, clear `sqlite_stat1`, and prove that exactly `files`, `symbols`, and `symbol_references` are analyzed once after candidate-index deferral but before identity SQL starts. At the start hook, neither those tables nor deferred query indexes may have statistics and the candidate reverse index must already be absent; after completion, TypeScript-deferred query indexes must have statistics while the candidate table/index remains unanalyzed. Cover both the TypeScript-deferred CLI lifecycle and a C#-only direct graph lifecycle, plus fresh MCP. Rebuilds, existing/update paths, symbols-only or disabled guards, and a false fresh flag must emit no statistics phase. Outer rollback must remove the new statistics; a non-cancellation SQLite failure must roll back sentinel/statistics writes inside the nested savepoint and continue graph resolution, while cancellation must propagate. Keep this phase on its dedicated hook and assert that the final `DbContext` maintenance hook is not invoked by the pre-graph refresh. -- High-churn update reference-index coverage should materialize exactly the production minimum target count, cross the 60% boundary with pure decision cases, and assert candidate-index deferral through the TypeScript-owned graph pass plus full restoration afterwards. Re-run the same high-cardinality scoped target set unchanged and prove that candidate deferral and graph refresh never start, so the existing candidate index is not rebuilt. Change exactly one target before invocation and prove that the raw target-count gate alone cannot stage indexes or force a full graph: the estimated mutating-target count must cross the same production threshold, while the authoritative file loop still persists the sparse change with a scoped graph refresh. A single duplicate-hardlink cleanup must likewise remain scoped instead of treating identity detection as preflight uncertainty. Fresh/rebuild CLI and fresh MCP TypeScript fixtures must also observe all four hotspot aggregate indexes absent during their qualifying bulk refresh and restored at completion. Existing-database MCP FTS bulk load must prove that the guard centrally forces a full graph plan; low-churn scoped updates must remain scoped. Do not lower the production boundary through a global testing override. +- Reference-secondary-index bulk-load coverage should assert the exact canonical schema at raw persistence, candidate deferral, identity start, candidate-lookup restoration, graph-required restore, mutual-recursion start, and final query restore. Raw persistence keeps the reverse candidate-symbol lookup while ordinary query/graph indexes are absent; an actual graph refresh drops it immediately before candidate deletion/materialization. Candidate construction completes while it is absent, then `candidate_lookup_restored` must restore it before resolution is rented or prepared. The ordinary middle graph set is exactly the candidate reverse lookup plus unresolved-folded, legacy NOCASE, and resolved reverse-edge after the graph-required boundary, and its bounded symbol-existence plan, three reverse-edge plans, and candidate reference-primary-key lookups must remain usable. When TypeScript augmentation owns the graph pass, assert that all ordinary graph/query indexes return before readiness, the reverse candidate-symbol lookup remains available through marker/grouping work, is absent only during candidate population, and is canonical again for resolution. Also prove direct `changes()` preservation, no-graph completion without candidate-index DDL, candidate-boundary cancellation/disposal recovery, abandoned-stage schema repair, and transactional rollback without relying on initialization repair. A guard still removes legacy single-prefix/all-row-mutual indexes without restoring them recoverably. +- Fresh bulk-load planner-statistics coverage should seed only the minimum file, symbol, and reference rows, clear `sqlite_stat1`, and prove that exactly `files`, `symbols`, and `symbol_references` are analyzed once after candidate-index deferral but before identity SQL starts. At the start hook, neither those tables nor deferred query indexes may have statistics and the candidate reverse index must already be absent; only after candidate construction may `candidate_lookup_restored` restore it. After completion, TypeScript-deferred query indexes must have statistics while the candidate table/index remains unanalyzed. Cover both the TypeScript-deferred CLI lifecycle and a C#-only direct graph lifecycle, plus fresh MCP. Rebuilds, existing/update paths, symbols-only or disabled guards, and a false fresh flag must emit no statistics phase. Outer rollback must remove the new statistics; a non-cancellation SQLite failure must roll back sentinel/statistics writes inside the nested savepoint and continue graph resolution, while cancellation must propagate. Keep this phase on its dedicated hook and assert that the final `DbContext` maintenance hook is not invoked by the pre-graph refresh. +- High-churn update reference-index coverage should materialize exactly the production minimum target count, cross the 60% boundary with pure decision cases, and assert candidate-index deferral only through TypeScript candidate population, restoration before resolution, and full canonical restoration afterwards. Re-run the same high-cardinality scoped target set unchanged and prove that candidate deferral and graph refresh never start, so the existing candidate index is not rebuilt. Change exactly one target before invocation and prove that the raw target-count gate alone cannot stage indexes or force a full graph: the estimated mutating-target count must cross the same production threshold, while the authoritative file loop still persists the sparse change with a scoped graph refresh. A single duplicate-hardlink cleanup must likewise remain scoped instead of treating identity detection as preflight uncertainty. Fresh/rebuild CLI and fresh MCP TypeScript fixtures must also observe all four hotspot aggregate indexes absent during their qualifying bulk refresh and restored at completion. Existing-database MCP FTS bulk load must prove that the guard centrally forces a full graph plan; low-churn scoped updates must remain scoped. Do not lower the production boundary through a global testing override. - Reference persistence binding coverage should exercise atomic and public transaction paths for both new-file inserts and replacement upserts. Assert 14 bound parameters per row, one ordinal slot per reference, one materialized ID per unique reference line, legacy `symbol_references.context` NULL storage, and intact normalized `reference_lines.context` text in one data-driven fixture. - Timeout-origin coverage should exercise timer cancellation and caller cancellation sequentially in one async test so the distinguishing assertion does not duplicate runner setup. - Bounded in-memory HTTP reads should cover unknown-length success and declared-length rejection in one async contract rather than splitting two adjacent buffer-policy assertions. @@ -1372,7 +1372,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `DbReaderCSharpUsingCatalogTests` では、reader scope の C# using catalog 契約を明示的に固定します。file-scoped namespace は EOF まで有効、nested namespace import は inclusive な最内 scope の外へ漏れず、同じ active path/line の反復 lookup は同一の cache set を再利用し、local/global alias の shadowing は解決可能な chain を保ちながら cycle で終了することを維持してください。 `CSharpBaseListParserTests` には SQLite を使わない base-list 構文 matrix をまとめ、generic constraint、nested generic / tuple / array、alias qualifier、primary / base constructor、宣言終端、および reader の type-reference 投影と metadata の head-identifier 投影の差を固定します。 maintenance lookup の coverage では `files(checksum)`、`files(path COLLATE NOCASE)`、`file_issues(file_id, kind)` の index を predicate と同期させ、checksum purge、ASCII case-alias lookup、再利用 stat の issue probe、directory/stem rename 候補が `EXPLAIN QUERY PLAN` で index `SEARCH` を使うことを必須とします。case-alias fixture は checksum で path lookup の欠落が隠れないよう content も変更し、SQLite `NOCASE` は Unicode casing contract ではなく ASCII prefilter にすぎないため managed 検証を authoritative に保ちます。wildcard を含む stem、拡張子なし、近似 stem の意味論は1つの fixture にまとめます。scoped cleanup planning の coverage では checksum と正確な同一 directory/stem の候補を昇順・重複排除済み ID snapshot に統合し、重複 plan の merge、plan 後に追加された一致 row を apply が取り込まないこと、apply 直前に planned ID を再読込して再出現した C# contract を clean retry まで延期することを固定します。grouped C# fixture は多数の target に共通 checksum/stem を与え、candidate reader の処理量が target 数の二乗ではなく unique key 数と返却 row 数に比例すること、および C# workspace 前の plan が一致する non-C# row を削除しないことを要求します。 - 参照解決 lookup の coverage では fresh schema / read migration の正確な index setを、file/name、保持したfolded/NOCASE composite prefix、解決済み逆辺、未解決mutual用partial pathまで同期させます。migrationと再open後に退役したsingle-prefix 6本と旧全row mutual indexが存在しないこと、列順・collation・partial predicateを厳密に検証し、full/scoped mutual SQLの両方で`EXPLAIN QUERY PLAN`のindex `SEARCH`を必須とします。fold-ready/legacy fallback双方のCLI/MCP exact-query signalを維持し、これらのindexを変える際は同名・別containerとpartial classが同一file候補より優先される既存ranking fixtureも再実行してください。 + 参照解決 lookup の coverage では fresh schema / read migration の正確な index setを、file/name、保持したfolded/NOCASE composite prefix、解決済み逆辺、未解決mutual用partial pathまで同期させます。migrationと再open後に退役したsingle-prefix 6本と旧全row mutual indexが存在しないこと、列順・collation・partial predicateを厳密に検証し、full/scoped mutual SQLの両方で`EXPLAIN QUERY PLAN`のindex `SEARCH`を必須とします。full / fresh / differential / retained の target-fact SQL は `idx_symbol_ref_candidates_symbol` の bounded な candidate-existence seek を使い、無関係な symbol を TEMP fact へ入れず、scoped SQL は dirty-candidate primary-key seek を維持してください。resolution SQL が `COUNT(DISTINCT)` set を持たず BINARY の `MIN(...) IS MAX(...)` を使い、empty、all-NULL、NULLと1 family、重複family、binary case variant、異なるfamily、物理NULL-key candidate 1件について旧aggregateと一致することを固定します。fold-ready/legacy fallback双方のCLI/MCP exact-query signalを維持し、これらのindexを変える際は同名・別containerとpartial classが同一file候補より優先される既存ranking fixtureも再実行してください。 repository metadata の graph coverage では TOML の local path と application manifest の assembly dependency を通常 writer 経由で index し、明示的な `project_reference` / `dependency` caller query を必須にして capability 広告と永続化済み graph のずれを防いでください。 reference identity writeのcoverageは、self-resolving reference 1件に列別SQLite audit triggerを設定します。安定rerunでsource・resolution・self・mutual rowのwriteが0、全4 phaseをcorruptした後にtarget resolutionを中断して先行source updateもrollback、復旧時はphaseごとにちょうど1 write、その後の安定rerunは再び0 writeであることを必須とします。 再利用 stat snapshot の意味論は、有効な1行、同じ stale extractor 言語の2行、NULL / text / integer / 不正 timestamp の stat storage を同居させ、malformed row の除外と stale-language 判定の false cache を1つの契約として固定します。 @@ -1678,9 +1678,9 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" plain-interface purge fixtureではsorted file-ID preflightを正確かつcancellation-awareに保ち、prepared cacheへ残さないSQLite parameter 500件以内のbatchに固定します。transition-path preflight の coverage は実memberを2つ目のpath batchだけに置き、path-first の `files(path)` と `symbols(file_id, kind)` のindex plan、keyword境界・大小文字decoyの拒否、cancellationによる中断、dynamic commandがprepared cacheへ入らないことを要求します。one-sided の同一stem/checksum rename と positive/unknown evidence の `--changed-between` 範囲外 fixture はimmutable cleanup planがreference再生成前にstale contract rowを消し、無関係なmissing non-C# rowを全掃除しないことを要求します。`Run_UpdateFiles_ChangedExistingRetainedTargetPreplansMatchingCsharpAlias` は既にindex済みでstat-changedのretained targetもchecksum一致のC# aliasをworkspace構築前にplanし、未変更targetは追加content readなしで永続checksumを再利用することを固定します。binary/oversized skip-record drift fixtureは`BuildSkippedFileRecord`後にmutationし、retry前にnested cleanup/upsert transactionとbatch markerの両方がrollbackされることを要求します。`Run_UpdateMode_WithChangedBetween_PurgesMissingIndexedPathOutsideDiff_4056` は authoritative false evidence のときだけ従来の全言語 reconciliation を維持することを別途固定します。`Run_UpdateMode_WithChangedBetween_CleanupPathReappearingAfterScanIsDeferredUntilRetry` はdiscovery中に削除済みcontractを復元し、`Run_UpdateCommits_ExactCleanupPathReappearingAfterSnapshotBarrierPreservesPriorRow` はfinal snapshot barrier後にexactなGit削除targetを復元します。どちらも旧rowとimplicit referenceを保持するpartial resultを要求します。expanded discovery のfatal errorとprepass後のchange/delete fixtureも旧C# rowとimplicit referenceを保持し、source evidenceをunknownにして、後続のclean retryで収束させます。C# mutation延期中も無関係なPython/JavaScript cleanupと未変更scoped targetは進行します。parallel symbol-cap fixtureではprepass後に初めて見つかったcontractをsource evidence unknownとして、後続のcomplete repairを強制します。 - case-folded cleanup coverageではSQLite `NOCASE` とmanaged foldingを候補filterだけに使い、leaf case・ancestor case・Unicode foldが一致してもfile identityが異なるpathは両方のexact rowを保持し、真のcase-only aliasだけがchecksum readなしで旧spellingを削除できることを固定します。無関係なhardlinkをaliasとするにはpathとidentityの両方が同じretained case-fold bucketに一致しなければならず、`ScopedFileCleanupReappearance_FoldBucketsDoNotCrossMatchTargetIdentities` は別target由来のpath一致とidentity一致の合成を防ぎます。Git name-status coverageはold/new pathにtabと改行を含む実NUL区切りrenameを使い、commit/range helperの両方がunquotedな正確な文字列を返すことを検証します。 - project-marker budget の integration coverage は directory budget を最小境界に override し、child を 1 件だけ列挙します。warning 伝播の検証だけのために本番の 8,192-directory cap を実体化しないでください。shared discovery の coverage は1回の directory enumeration から C#、VB、F#、MSBuild の全 fingerprint が得られることを検証し、family-scope coverage は scan 後の ancestor filesystem probe が0回、fingerprint budget 枯渇後も complete な scope snapshot を利用可能、不完全 discovery では live fallback を維持することを固定してください。 -- reference secondary-index の bulk-load coverage は raw persistence、candidate 遅延、identity 開始、graph-required 復元、mutual-recursion 開始、最終 query 復元の各境界で正確な canonical schema を検証してください。raw persistence 中は通常の query / graph index を外しても reverse candidate-symbol lookup を維持し、実際の graph refresh が candidate row を削除・構築する直前だけ外します。通常の中間 graph 集合は unresolved-folded、legacy NOCASE、resolved reverse-edge の正確に3本であり、candidate の reference-primary-key lookup を保ったまま3経路の plan が利用可能でなければなりません。TypeScript augmentation が graph pass を担当する場合は、readiness 前に通常の graph / query index を復元し、marker / grouping 中は reverse candidate-symbol lookup を維持し、candidate 構築から mutual recursion の完了までだけ不在にします。transactional full scan では readiness work 後かつ outer full-scan transaction の commit 前の復元を、recoverable scoped update / MCP indexing では readiness transaction の commit までの guard ownership 保持とその直後の復元を固定します。直接の `changes()` 保持、candidate index DDL を伴わない graph 不要時の完了、graph 段階での cancellation / dispose recovery、途中終了後の schema repair、initialization repair に依存しない transactional rollback も証明してください。guard は legacy single-prefix / 全row mutual index も削除し、recoverable mode では復元しません。 -- fresh bulk-load の planner-statistics coverage は必要最小限の file / symbol / reference row だけを seed し、`sqlite_stat1` を消去して、candidate index の遅延後かつ identity SQL の開始前に `files`、`symbols`、`symbol_references` の正確に3 tableだけを1回 ANALYZE することを証明してください。開始 hook では対象 table と deferred query index の統計がまだ存在せず、candidate reverse index は既に不在でなければなりません。完了後は TypeScript-deferred query index に統計があり、candidate table / index は未解析のままであることを固定します。TypeScript-deferred CLI lifecycle、C#-only direct graph lifecycle、fresh MCP のすべてを対象にしてください。rebuild、既存 / update 経路、symbols-only、guard 無効、fresh flag false では statistics phase を発生させません。outer rollback は新しい統計を消去し、cancellation 以外の SQLite failure は nested savepoint 内の sentinel / statistics write を rollback して graph resolution を継続し、cancellation は伝播しなければなりません。この phase は専用 hook に限定し、pre-graph refresh が最終 `DbContext` maintenance hook を呼ばないことも検証してください。 -- high-churn update の reference-index coverage は production の最小 target 数だけを実体化し、pure decision case で60%境界を越え、TypeScript-owned graph pass の完了まで candidate index が遅延し、最後に全 index が復元されることを検証してください。同じ高 cardinality scoped target 集合を unchanged のまま再実行し、candidate 遅延と graph refresh が始まらず既存 candidate index を再構築しないことも証明します。実行前に target を正確に1件だけ変更し、raw target-count gate だけでは index staging や full graph 強制に入らず、変更を起こし得る target の見積もり件数が同じ production threshold を超える必要があること、authoritative file loop が疎な変更を scoped graph refresh で永続化することも固定します。また、duplicate-hardlink cleanup が1件だけなら identity detection を preflight の不確実性として扱わず、scoped のままであることも固定します。fresh / rebuild CLI と fresh MCP の TypeScript fixture では、条件を満たす bulk refresh 中に hotspot aggregate の4 indexも不在で、完了後に復元されることを観測します。既存DBに対する MCP FTS bulk load では guard が full graph plan を中央強制することを証明し、low-churn scoped update は scoped のまま維持してください。global testing override で production 境界を下げないでください。 +- reference secondary-index の bulk-load coverage は raw persistence、candidate 遅延、identity 開始、candidate lookup 復元、graph-required 復元、mutual-recursion 開始、最終 query 復元の各境界で正確な canonical schema を検証してください。raw persistence 中は通常の query / graph index を外しても reverse candidate-symbol lookup を維持し、実際の graph refresh が candidate row を削除・構築する直前だけ外します。candidate 構築中は不在とし、完了直後の `candidate_lookup_restored` で resolution の rent / prepare 前に復元します。通常の中間 graph 集合は candidate reverse lookup に unresolved-folded、legacy NOCASE、resolved reverse-edge の3本を加えた集合であり、bounded symbol-existence plan、3経路のreverse-edge plan、candidate のreference-primary-key lookup が利用可能でなければなりません。TypeScript augmentation が graph pass を担当する場合は、readiness 前に通常の graph / query index を復元し、marker / grouping 中は reverse candidate-symbol lookup を維持し、candidate 構築中だけ不在、resolution では canonical に戻ることを固定します。直接の `changes()` 保持、candidate index DDL を伴わない graph 不要時の完了、candidate 境界での cancellation / dispose recovery、途中終了後の schema repair、initialization repair に依存しない transactional rollback も証明してください。guard は legacy single-prefix / 全row mutual index も削除し、recoverable mode では復元しません。 +- fresh bulk-load の planner-statistics coverage は必要最小限の file、symbol、reference row だけを seed し、`sqlite_stat1` を消去して、candidate index の遅延後かつ identity SQL の開始前に `files`、`symbols`、`symbol_references` の正確に3 tableだけを1回 ANALYZE することを証明してください。開始 hook では対象 table と deferred query index の統計がまだ存在せず、candidate reverse index は既に不在でなければなりません。candidate 構築後にだけ `candidate_lookup_restored` で復元し、完了後は TypeScript-deferred query index に統計がある一方、candidate table / index は未解析のままであることを固定します。TypeScript-deferred CLI lifecycle、C#-only direct graph lifecycle、fresh MCP のすべてを対象にしてください。rebuild、既存 / update 経路、symbols-only、guard 無効、fresh flag false では statistics phase を発生させません。outer rollback は新しい統計を消去し、cancellation 以外の SQLite failure は nested savepoint 内の sentinel / statistics write を rollback して graph resolution を継続し、cancellation は伝播しなければなりません。この phase は専用 hook に限定し、pre-graph refresh が最終 `DbContext` maintenance hook を呼ばないことも検証してください。 +- high-churn update の reference-index coverage は production の最小 target 数だけを実体化し、pure decision case で60%境界を越え、TypeScript candidate 構築中だけ candidate index を遅延し、resolution 前の復元と完了後の canonical schema を検証してください。同じ高 cardinality scoped target 集合を unchanged のまま再実行し、candidate 遅延と graph refresh が始まらず既存 candidate index を再構築しないことも証明します。実行前に target を正確に1件だけ変更し、raw target-count gate だけでは index staging や full graph 強制に入らず、変更を起こし得る target の見積もり件数が同じ production threshold を超える必要があること、authoritative file loop が疎な変更を scoped graph refresh で永続化することも固定します。また、duplicate-hardlink cleanup が1件だけなら identity detection を preflight の不確実性として扱わず、scoped のままであることも固定します。fresh / rebuild CLI と fresh MCP の TypeScript fixture では、条件を満たす bulk refresh 中に hotspot aggregate の4 indexも不在で、完了後に復元されることを観測します。既存DBに対する MCP FTS bulk load では guard が full graph plan を中央強制することを証明し、low-churn scoped update は scoped のまま維持してください。global testing override で production 境界を下げないでください。 - reference persistence binding の coverage は、新規 file insert と replacement upsert の両方について atomic 経路と public transaction 経路を検証してください。1 row あたり14 bound parameter、reference ごとに1つの ordinal slot、unique reference line ごとに1つの materialized ID、legacy `symbol_references.context` の NULL、正規化済み `reference_lines.context` text の保持を1つの data-driven fixture で固定してください。 - search guard と LSP の candidate/materialization cap coverage は、各 active limit を sentinel 1 件だけで超えます。利用可能な production constant は再利用し、境界を観測できた後に数百件の余分な row や symbol を残さないでください。 - `LspServerTests.TryReadAllPositionLinesFromFile_PreservesUtf8AndRejectsGrowthAfterLengthCheck_Issue4750` は、決定的な length-check hook の直後に UTF-8 を 1 回 append して position-file byte cap を超えます。multibyte / control の成功 case も同じ fixture に保ち、chunk 境界の decode と同時増大の拒否で上限付き setup を共有してください。 diff --git a/changelog.d/unreleased/+initial-full-index-candidate-resolution.changed.md b/changelog.d/unreleased/+initial-full-index-candidate-resolution.changed.md new file mode 100644 index 0000000000..1da44167a7 --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-candidate-resolution.changed.md @@ -0,0 +1,17 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs + - src/CodeIndex/Database/DbWriter.References.cs + - src/CodeIndex/Database/DbWriter.cs + - src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs + - src/CodeIndex/Database/ReferenceSecondaryIndexSql.cs +--- + +## English + +- **Cold reference resolution limits target-key work to candidates** — Graph finalization now restores the candidate reverse index immediately after candidate insertion, prepares resolution separately, materializes target-family facts only for candidate-bearing symbols, and replaces per-group DISTINCT sets with equivalent BINARY min/max singleton checks while preserving legacy NULL and ambiguity semantics. + +## 日本語 + +- **初回 reference resolution の target-key 処理を candidate に限定** — graph finalization は candidate insert 直後に reverse index を復元して resolution を別 command で prepare し、candidate を持つ symbol だけの target-family fact を materialize します。legacy NULL と ambiguity semantics を維持したまま、group ごとの DISTINCT set を等価な BINARY min/max singleton 判定へ置き換えます。 diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index d494b91aa5..1be16420f6 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -88,9 +88,15 @@ CROSS JOIN symbols AS s INDEXED BY idx_symbols_name_folded AND target_file.lang = dirty_name.lang AND target_file.lang <> 'ambiguous_m' GROUP BY target_file.lang, s.name_folded - HAVING COUNT(DISTINCT target_file.path || char(31) || - COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || - COALESCE(s.name, '')) = 1; + HAVING COUNT(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) > 0 + AND MIN(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) + IS MAX(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')); -- Keep the scoped projection aligned with the full-refresh union-wide -- uniqueness contract for callers whose .m dialect is unresolved. @@ -109,9 +115,15 @@ CROSS JOIN symbols AS s INDEXED BY idx_symbols_name_folded AND s.name_folded = dirty_name.name_folded AND target_file.lang IN ('matlab', 'objc') GROUP BY s.name_folded - HAVING COUNT(DISTINCT target_file.path || char(31) || - COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || - COALESCE(s.name, '')) = 1; + HAVING COUNT(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) > 0 + AND MIN(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) + IS MAX(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')); """; private static readonly string RefreshScopedReferenceCandidatesSql = BuildScopedReferenceCandidatesSql(); diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 2e404039bf..763cd041dc 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -236,7 +236,13 @@ ON csharp_reference_facts(reference_id) SELECT target.id, {BuildReferenceResolutionTargetKeySql()} FROM symbols AS target - JOIN files AS target_file ON target_file.id = target.file_id; + JOIN files AS target_file ON target_file.id = target.file_id + WHERE EXISTS ( + SELECT 1 + FROM symbol_reference_candidates AS candidate + INDEXED BY idx_symbol_ref_candidates_symbol + WHERE candidate.symbol_id = target.id + ); """; private static string BuildRefreshCSharpReferenceFactsSql(string scopePredicate) @@ -1134,9 +1140,15 @@ FROM symbols AS s WHERE s.name_folded IS NOT NULL AND target_file.lang <> 'ambiguous_m' GROUP BY target_file.lang, s.name_folded - HAVING COUNT(DISTINCT target_file.path || char(31) || - COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || - COALESCE(s.name, '')) = 1; + HAVING COUNT(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) > 0 + AND MIN(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) + IS MAX(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')); -- An ambiguous .m caller can bind to either dialect, so uniqueness must hold -- across the MATLAB/Objective-C union rather than within either language alone. @@ -1153,9 +1165,15 @@ FROM symbols AS s WHERE s.name_folded IS NOT NULL AND target_file.lang IN ('matlab', 'objc') GROUP BY s.name_folded - HAVING COUNT(DISTINCT target_file.path || char(31) || - COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || - COALESCE(s.name, '')) = 1; + HAVING COUNT(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) > 0 + AND MIN(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) + IS MAX(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')); """; private static string RefreshReferenceCandidatesSql => $""" @@ -1468,7 +1486,9 @@ FROM csharp_type_reference_members AS type_member GROUP BY type_member.name_folded, type_member.name, type_member.type_arity - HAVING COUNT(DISTINCT type_member.type_identity COLLATE BINARY) = 1 + HAVING COUNT(type_member.type_identity) > 0 + AND MIN(type_member.type_identity COLLATE BINARY) + IS MAX(type_member.type_identity COLLATE BINARY) ), matched_csharp_type_reference_families( reference_id, @@ -1661,7 +1681,9 @@ FROM csharp_instantiation_type_members AS type_member GROUP BY type_member.name_folded, type_member.name COLLATE BINARY, type_member.type_arity - HAVING COUNT(DISTINCT type_member.type_identity COLLATE BINARY) = 1 + HAVING COUNT(type_member.type_identity) > 0 + AND MIN(type_member.type_identity COLLATE BINARY) + IS MAX(type_member.type_identity COLLATE BINARY) ), csharp_instantiation_constructor_members( symbol_id, @@ -1886,18 +1908,24 @@ FROM temp.{ReferenceLowerRankCandidateMatchesTable} AS lower_rank_match private const string ReferenceResolutionValueSql = """ ( SELECT CASE WHEN candidate_count = 1 THEN minimum_symbol_id END, - CASE WHEN target_family_count = 1 THEN minimum_target_key END, + CASE WHEN has_single_target_family = 1 THEN minimum_target_key END, candidate_count, CASE WHEN candidate_count = 0 THEN 'unresolved' WHEN candidate_count = 1 THEN 'resolved' - WHEN target_family_count = 1 THEN 'resolved_group' + WHEN has_single_target_family = 1 THEN 'resolved_group' ELSE 'ambiguous' END FROM ( SELECT COUNT(*) AS candidate_count, MIN(c.symbol_id) AS minimum_symbol_id, - COUNT(DISTINCT target_fact.target_key) AS target_family_count, + CASE + WHEN COUNT(target_fact.target_key) > 0 + AND MIN(target_fact.target_key COLLATE BINARY) + IS MAX(target_fact.target_key COLLATE BINARY) + THEN 1 + ELSE 0 + END AS has_single_target_family, MIN(target_fact.target_key) AS minimum_target_key FROM symbol_reference_candidates AS c JOIN temp.reference_resolution_symbol_facts AS target_fact @@ -1948,7 +1976,13 @@ WITH resolution_facts AS MATERIALIZED ( SELECT candidate.reference_id, COUNT(*) AS candidate_count, MIN(candidate.symbol_id) AS minimum_symbol_id, - COUNT(DISTINCT target_fact.target_key) AS target_family_count, + CASE + WHEN COUNT(target_fact.target_key) > 0 + AND MIN(target_fact.target_key COLLATE BINARY) + IS MAX(target_fact.target_key COLLATE BINARY) + THEN 1 + ELSE 0 + END AS has_single_target_family, MIN(target_fact.target_key) AS minimum_target_key FROM symbol_reference_candidates AS candidate JOIN temp.{ReferenceResolutionSymbolFactsTable} AS target_fact @@ -1960,12 +1994,13 @@ UPDATE symbol_references AS r WHEN resolution.candidate_count = 1 THEN resolution.minimum_symbol_id END, target_symbol_key = CASE - WHEN resolution.target_family_count = 1 THEN resolution.minimum_target_key + WHEN resolution.has_single_target_family = 1 + THEN resolution.minimum_target_key END, resolution_candidate_count = resolution.candidate_count, resolution_state = CASE WHEN resolution.candidate_count = 1 THEN 'resolved' - WHEN resolution.target_family_count = 1 THEN 'resolved_group' + WHEN resolution.has_single_target_family = 1 THEN 'resolved_group' ELSE 'ambiguous' END, is_self_reference = CASE @@ -2026,9 +2061,9 @@ internal static void RebuildRetainedReferenceGraph( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - using var command = connection.CreateCommand(); - command.Transaction = transaction; - command.CommandText = + using var candidateCommand = connection.CreateCommand(); + candidateCommand.Transaction = transaction; + candidateCommand.CommandText = CreateReferenceUniqueFamiliesSql + ";\n" + CreateCSharpReferenceFactIndexesSql + ";\n" + RefreshReferenceSourceSymbolsFullSql + ";\n" + @@ -2039,12 +2074,25 @@ internal static void RebuildRetainedReferenceGraph( RefreshCSharpPropertyTargetFactsFullSql + "\n" + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceUniqueFamiliesSql + "\n" + - RefreshReferenceCandidatesSql + "\n" + + RefreshReferenceCandidatesSql; + using (var candidateCancellationRegistration = + cancellationToken.Register(candidateCommand.Cancel)) + { + candidateCommand.ExecuteNonQuery(); + } + cancellationToken.ThrowIfCancellationRequested(); + ReferenceCandidateRefreshCompletedForTesting?.Invoke(); + cancellationToken.ThrowIfCancellationRequested(); + + using var resolutionCommand = connection.CreateCommand(); + resolutionCommand.Transaction = transaction; + resolutionCommand.CommandText = RefreshReferenceResolutionSymbolFactsFullSql + "\n" + RefreshReferenceResolutionFullSql + "\n" + RefreshMutualRecursionFlagsSql; - using var cancellationRegistration = cancellationToken.Register(command.Cancel); - command.ExecuteNonQuery(); + using var resolutionCancellationRegistration = + cancellationToken.Register(resolutionCommand.Cancel); + resolutionCommand.ExecuteNonQuery(); cancellationToken.ThrowIfCancellationRequested(); } @@ -2950,7 +2998,8 @@ internal void RefreshMutualRecursionFlags( graphScope.IsCompleting = true; SqliteCommand? createUniqueFamiliesCommand = null; SqliteCommand? createCSharpReferenceFactIndexesCommand = null; - SqliteCommand? refreshIdentityCommand = null; + SqliteCommand? refreshCandidateCommand = null; + SqliteCommand? refreshResolutionCommand = null; SqliteCommand? refreshMutualCommand = null; try { @@ -2987,7 +3036,8 @@ internal void RefreshMutualRecursionFlags( // 真に空のdatabaseではcandidate側resolution factsを1回集約し、candidateを持つ // referenceだけをseekする。その他の既存resolutionはstable rowを書き換えない // differential pathを維持する。 - string refreshIdentitySql; + string refreshCandidateSql; + string refreshResolutionSql; if (refreshPlan.UseFullRefresh) { var hasPersistedReferenceResolutionState = !useFreshReferenceResolutionDefaults @@ -3000,7 +3050,7 @@ internal void RefreshMutualRecursionFlags( : hasPersistedReferenceResolutionState ? RefreshReferenceResolutionDifferentialSql : RefreshReferenceResolutionFullSql; - refreshIdentitySql = + refreshCandidateSql = (refreshReferenceSourcesSql == null ? string.Empty : refreshReferenceSourcesSql + ";\n") + @@ -3011,14 +3061,15 @@ internal void RefreshMutualRecursionFlags( RefreshCSharpPropertyTargetFactsFullSql + "\n" + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceUniqueFamiliesSql + "\n" + - RefreshReferenceCandidatesSql + "\n" + - RefreshReferenceResolutionSymbolFactsFullSql + "\n" + - refreshReferenceResolutionSql + "\n"; + RefreshReferenceCandidatesSql + "\n"; + refreshResolutionSql = + RefreshReferenceResolutionSymbolFactsFullSql + "\n" + + refreshReferenceResolutionSql + "\n"; } else { DeleteRemovedReferenceCandidates(cancellationToken); - refreshIdentitySql = RefreshScopedReferenceSourceSymbolsSql + "\n" + + refreshCandidateSql = RefreshScopedReferenceSourceSymbolsSql + "\n" + RefreshCSharpReferenceFactsScopedSql + "\n" + RefreshCSharpSymbolFactsScopedSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + @@ -3026,15 +3077,16 @@ internal void RefreshMutualRecursionFlags( RefreshCSharpPropertyTargetFactsScopedSql + "\n" + NormalizeCSharpPropertyReceiverReferencesScopedSql + "\n" + RefreshScopedReferenceUniqueFamiliesSql + "\n" + - RefreshScopedReferenceCandidatesSql + "\n" + - RefreshScopedReferenceResolutionSymbolFactsSql + "\n" + - RefreshScopedReferenceResolutionSql + "\n" + - ExpandReferenceGraphNewMutualScopeSql + "\n"; + RefreshScopedReferenceCandidatesSql + "\n"; + refreshResolutionSql = + RefreshScopedReferenceResolutionSymbolFactsSql + "\n" + + RefreshScopedReferenceResolutionSql + "\n" + + ExpandReferenceGraphNewMutualScopeSql + "\n"; } var hotspotReferenceFileIds = GetReferenceGraphRefreshFileIds( refreshPlan.UseFullRefresh, cancellationToken); - refreshIdentityCommand = RentCommand(refreshIdentitySql, static _ => { }); + refreshCandidateCommand = RentCommand(refreshCandidateSql, static _ => { }); // Reconcile the marker inside the same transaction, but before the graph refresh // so the public SQLite changes() result continues to describe recursion updates. // High-level indexing defers v7 while untouched legacy C# family rows remain. @@ -3046,7 +3098,19 @@ internal void RefreshMutualRecursionFlags( ClearReferenceIdentityContractReady(); cancellationToken.ThrowIfCancellationRequested(); referenceSecondaryIndexBulkLoad?.ReportIdentityRefreshStarted(); - refreshIdentityCommand.ExecuteNonQuery(); + refreshCandidateCommand.ExecuteNonQuery(); + cancellationToken.ThrowIfCancellationRequested(); + ReferenceCandidateRefreshCompletedForTesting?.Invoke(); + cancellationToken.ThrowIfCancellationRequested(); + referenceSecondaryIndexBulkLoad?.PrepareForReferenceResolution(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + // Rent only after the reverse candidate index is restored. SQLite prepares a + // cached command on first use, so this ordering keeps the candidate-symbol EXISTS + // probe on its bounded (symbol_id, reference_id) lookup plan. + // candidate逆引きindexの復元後に初めてRentし、初回prepareから + // (symbol_id, reference_id)のbounded EXISTS lookup planを選ばせる。 + refreshResolutionCommand = RentCommand(refreshResolutionSql, static _ => { }); + refreshResolutionCommand.ExecuteNonQuery(); cancellationToken.ThrowIfCancellationRequested(); // Resolution changes alter the default C# common-call hotspot projection even // when the caller file itself was skipped. Refresh those source-file aggregates @@ -3079,8 +3143,10 @@ internal void RefreshMutualRecursionFlags( { if (refreshMutualCommand != null) ReleaseCommand(refreshMutualCommand); - if (refreshIdentityCommand != null) - ReleaseCommand(refreshIdentityCommand); + if (refreshResolutionCommand != null) + ReleaseCommand(refreshResolutionCommand); + if (refreshCandidateCommand != null) + ReleaseCommand(refreshCandidateCommand); if (createCSharpReferenceFactIndexesCommand != null) ReleaseCommand(createCSharpReferenceFactIndexesCommand); if (createUniqueFamiliesCommand != null) diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 45597b64a5..f80d31856e 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -38,6 +38,7 @@ public partial class DbWriter private static readonly AsyncLocal?> ScopedBatchRowSkipWarningForTesting = new(); private static readonly AsyncLocal?> ScopedBatchProgressCheckpointForTesting = new(); private static readonly AsyncLocal ScopedMutualRecursionRefreshForTesting = new(); + private static readonly AsyncLocal ScopedReferenceCandidateRefreshCompletedForTesting = new(); private static readonly AsyncLocal ScopedCSharpContractPreflightForTesting = new(); private static readonly AsyncLocal ScopedCSharpContractWorkspaceReadForTesting = new(); private static readonly AsyncLocal?> @@ -95,6 +96,12 @@ internal static Action? MutualRecursionRefreshForTesting set => ScopedMutualRecursionRefreshForTesting.Value = value; } + internal static Action? ReferenceCandidateRefreshCompletedForTesting + { + get => ScopedReferenceCandidateRefreshCompletedForTesting.Value; + set => ScopedReferenceCandidateRefreshCompletedForTesting.Value = value; + } + internal static Action? CSharpContractPreflightForTesting { get => ScopedCSharpContractPreflightForTesting.Value; diff --git a/src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs b/src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs index d29bba5e3d..3f68d83342 100644 --- a/src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs +++ b/src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs @@ -4,7 +4,8 @@ namespace CodeIndex.Database; /// /// Temporarily removes reference-query and graph indexes while raw rows are populated, then -/// removes the candidate reverse lookup only when graph candidate materialization begins. +/// removes the candidate reverse lookup only while graph candidates are materialized. The +/// reverse lookup is restored before candidate-backed resolution facts are prepared. /// File and reference-line maintenance indexes normally remain available; an authoritative /// empty-database CLI transaction may defer its two persistence-only probes until graph work. /// @@ -171,6 +172,24 @@ internal void PrepareForMutualRecursion(CancellationToken cancellationToken = de writer.RestoreGraphFinalizationRequiredReferenceSecondaryIndexes(cancellationToken); } + /// + /// Restore the candidate reverse lookup after candidate population and before the + /// separately prepared resolution command. Full resolution can then materialize target + /// keys only for candidate-bearing symbols using bounded reverse-index existence seeks. + /// candidate生成後、別commandのresolutionをprepareする前に逆引きindexを復元する。 + /// これによりfull resolution factはboundedな逆引き存在seekを使い、candidateを持つ + /// symbolだけのtarget keyをmaterializeする。 + /// + internal void PrepareForReferenceResolution(CancellationToken cancellationToken = default) + { + var writer = _writer; + if (writer == null) + return; + + RestoreAuthoritativeFreshPersistenceIndexes(writer, cancellationToken); + writer.RestoreCandidateResolutionReferenceSecondaryIndexes(cancellationToken); + } + /// /// Restore every deferred index except the candidate reverse lookup. A later graph /// refresh can then populate candidates without maintaining that B-tree, while readiness @@ -309,6 +328,15 @@ internal void RestoreGraphFinalizationRequiredReferenceSecondaryIndexes( ReportReferenceSecondaryIndexBulkLoadState("graph_required_restored"); } + internal void RestoreCandidateResolutionReferenceSecondaryIndexes( + CancellationToken cancellationToken = default) + { + RestoreReferenceSecondaryIndexes( + ReferenceSecondaryIndexSql.CandidatePopulationDeferred, + cancellationToken); + ReportReferenceSecondaryIndexBulkLoadState("candidate_lookup_restored"); + } + internal void RestoreReferenceSecondaryIndexesForDeferredGraph( CancellationToken cancellationToken = default) { diff --git a/src/CodeIndex/Database/ReferenceSecondaryIndexSql.cs b/src/CodeIndex/Database/ReferenceSecondaryIndexSql.cs index 648437e8eb..70ec7aacfa 100644 --- a/src/CodeIndex/Database/ReferenceSecondaryIndexSql.cs +++ b/src/CodeIndex/Database/ReferenceSecondaryIndexSql.cs @@ -10,8 +10,9 @@ internal readonly record struct ReferenceSecondaryIndexDefinition( /// The raw-persistence set normally stays available while bulk extraction is writing rows; /// an authoritative empty-database CLI transaction may defer its two persistence-only /// probes. The candidate reverse lookup is dropped only when candidate materialization -/// begins; the graph-finalization set is restored immediately before mutual-recursion -/// evaluation, and the remaining query set is restored after graph finalization completes. +/// begins and restored before candidate-backed resolution facts are prepared; the graph- +/// finalization set is restored immediately before mutual-recursion evaluation, and the +/// remaining query set is restored after graph finalization completes. /// internal static class ReferenceSecondaryIndexSql { @@ -120,9 +121,9 @@ internal static class ReferenceSecondaryIndexSql private static readonly ReferenceSecondaryIndexDefinition[] CandidatePopulationDeferredDefinitions = [ - // Candidate materialization and resolution use the primary key's reference_id - // prefix. Defer the reverse symbol lookup so bulk graph refresh can populate the - // candidate table without maintaining a second B-tree row by row. + // Candidate materialization uses the primary key's reference_id prefix. Defer the + // reverse symbol lookup while populating the table, then restore it so resolution + // facts can test candidate-bearing symbols with bounded symbol_id existence seeks. new( "idx_symbol_ref_candidates_symbol", "CREATE INDEX IF NOT EXISTS idx_symbol_ref_candidates_symbol ON symbol_reference_candidates(symbol_id, reference_id)"), diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index ec19494085..bfe30ab2e0 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -387,9 +387,18 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() sql, StringComparison.Ordinal); Assert.Contains( - "HAVING COUNT(DISTINCT type_member.type_identity COLLATE BINARY) = 1", + "HAVING COUNT(type_member.type_identity) > 0", sql, StringComparison.Ordinal); + Assert.Contains( + "MIN(type_member.type_identity COLLATE BINARY)", + sql, + StringComparison.Ordinal); + Assert.Contains( + "IS MAX(type_member.type_identity COLLATE BINARY)", + sql, + StringComparison.Ordinal); + Assert.DoesNotContain("COUNT(DISTINCT", sql, StringComparison.Ordinal); Assert.DoesNotContain("symbols AS other_type", sql, StringComparison.Ordinal); Assert.DoesNotContain("file-local:", sql, StringComparison.Ordinal); Assert.DoesNotContain("ranked_constructor_owners", sql, StringComparison.Ordinal); @@ -1958,6 +1967,28 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() detail.Equals("SCAN type_symbol", StringComparison.OrdinalIgnoreCase) || detail.StartsWith("SCAN type_symbol ", StringComparison.OrdinalIgnoreCase)); + var fullResolutionFacts = Assert.Single( + DbWriter.ReferenceResolutionFactSqlForTesting, + static entry => entry.Scope == "full"); + var fullResolutionFactInsert = Assert.Single( + fullResolutionFacts.MaterializationSql + .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Where(static statement => statement.StartsWith( + "INSERT INTO temp.reference_resolution_symbol_facts", + StringComparison.Ordinal))); + var fullResolutionFactPlan = ReadQueryPlanDetails( + _db.Connection, + fullResolutionFactInsert); + Assert.Contains(fullResolutionFactPlan, static detail => detail.Contains( + "SEARCH candidate USING COVERING INDEX idx_symbol_ref_candidates_symbol", + StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(fullResolutionFactPlan, static detail => + detail.Equals("SCAN candidate", StringComparison.OrdinalIgnoreCase) + || detail.StartsWith("SCAN candidate ", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(fullResolutionFactPlan, static detail => detail.Contains( + "USE TEMP B-TREE", + StringComparison.OrdinalIgnoreCase)); + var scopedResolutionFacts = Assert.Single( DbWriter.ReferenceResolutionFactSqlForTesting, static entry => entry.Scope == "scoped"); @@ -2761,6 +2792,175 @@ FROM symbol_references Assert.Equal("resolved", ReadReferenceResolutionState(fileId)); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RefreshReferenceIdentities_CancellationAfterCandidatesRollsBackAndRetries( + bool forceFullRefresh) + { + var fileId = UpsertTestFileWithLanguage( + "src/candidate-boundary.py", + "python", + $"candidate-boundary-{forceFullRefresh}"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "CandidateBoundary", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "CandidateBoundary", + ReferenceKind = "call", + Line = 10, + Column = 1, + Context = "CandidateBoundary();", + }, + ], refreshMutualRecursionFlags: false); + _writer.RefreshMutualRecursionFlags(); + Assert.Equal(1, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + Assert.Equal("resolved", ReadReferenceResolutionState(fileId)); + + using var scope = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: forceFullRefresh); + using (var transaction = _writer.BeginTransaction()) + { + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "CandidateBoundary", + Line = 2, + StartLine = 2, + EndLine = 2, + }, + ]); + transaction.Commit(); + } + + var previousHook = DbWriter.ReferenceCandidateRefreshCompletedForTesting; + using var cancellation = new CancellationTokenSource(); + var boundaryCount = 0; + try + { + DbWriter.ReferenceCandidateRefreshCompletedForTesting = () => + { + boundaryCount++; + Assert.Equal( + 2, + ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + cancellation.Cancel(); + previousHook?.Invoke(); + }; + + var exception = Assert.Throws(() => + _writer.RefreshMutualRecursionFlags(cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(1, boundaryCount); + Assert.Equal(1, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + Assert.Equal("resolved", ReadReferenceResolutionState(fileId)); + } + finally + { + DbWriter.ReferenceCandidateRefreshCompletedForTesting = previousHook; + } + + _writer.RefreshMutualRecursionFlags(); + Assert.Equal(2, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + Assert.Equal("resolved_group", ReadReferenceResolutionState(fileId)); + } + + [Fact] + public void RebuildRetainedReferenceGraph_CancellationAfterCandidatesRollsBackAndRetries() + { + var fileId = UpsertTestFileWithLanguage( + "src/retained-candidate-boundary.py", + "python", + "retained-candidate-boundary"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "RetainedBoundary", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "RetainedBoundary", + ReferenceKind = "call", + Line = 10, + Column = 1, + Context = "RetainedBoundary();", + }, + ], refreshMutualRecursionFlags: false); + _writer.RefreshMutualRecursionFlags(); + Assert.Equal(1, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "RetainedBoundary", + Line = 2, + StartLine = 2, + EndLine = 2, + }, + ]); + + var previousHook = DbWriter.ReferenceCandidateRefreshCompletedForTesting; + using var cancellation = new CancellationTokenSource(); + try + { + DbWriter.ReferenceCandidateRefreshCompletedForTesting = () => + { + Assert.Equal( + 2, + ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + cancellation.Cancel(); + previousHook?.Invoke(); + }; + using var transaction = _db.Connection.BeginTransaction(); + var exception = Assert.Throws(() => + DbWriter.RebuildRetainedReferenceGraph( + _db.Connection, + transaction, + cancellation.Token)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + } + finally + { + DbWriter.ReferenceCandidateRefreshCompletedForTesting = previousHook; + } + + Assert.Equal(1, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + using (var retry = _db.Connection.BeginTransaction()) + { + DbWriter.RebuildRetainedReferenceGraph( + _db.Connection, + retry, + CancellationToken.None); + retry.Commit(); + } + Assert.Equal(2, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + Assert.Equal("resolved_group", ReadReferenceResolutionState(fileId)); + } + [Fact] public void ReferenceGraph_NimStyleInsensitiveIdentityResolvesAndSearches_Issue4738() { diff --git a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs index fef6946785..40c6f6cb51 100644 --- a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs +++ b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs @@ -275,7 +275,12 @@ public void FreshResolutionSql_MaterializesCandidateFactsWithoutOuterReferenceSc Assert.Contains("FROM resolution_facts AS resolution", sql, StringComparison.Ordinal); Assert.Contains("WHERE r.id = resolution.reference_id", sql, StringComparison.Ordinal); Assert.Contains("is_self_reference = CASE", sql, StringComparison.Ordinal); - Assert.DoesNotContain("WHERE EXISTS", sql, StringComparison.Ordinal); + Assert.Contains( + "FROM symbol_reference_candidates AS candidate", + sql, + StringComparison.Ordinal); + Assert.Contains("INDEXED BY idx_symbol_ref_candidates_symbol", sql, StringComparison.Ordinal); + Assert.Contains("WHERE EXISTS", sql, StringComparison.Ordinal); Assert.Equal(1, CountOccurrences(sql, "UPDATE symbol_references AS r")); } @@ -302,6 +307,9 @@ public void ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshSc "JOIN temp.reference_resolution_symbol_facts AS target_fact", resolutionSql, StringComparison.Ordinal); + Assert.DoesNotContain("COUNT(DISTINCT", resolutionSql, StringComparison.Ordinal); + Assert.Contains("MIN(target_fact.target_key COLLATE BINARY)", resolutionSql, StringComparison.Ordinal); + Assert.Contains("IS MAX(target_fact.target_key COLLATE BINARY)", resolutionSql, StringComparison.Ordinal); Assert.Equal( scope is "differential" or "scoped" ? 2 : 1, CountOccurrences(resolutionSql, "JOIN temp.reference_resolution_symbol_facts AS target_fact")); @@ -320,13 +328,124 @@ public void ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshSc StringComparison.Ordinal); Assert.Contains("GROUP BY candidate.symbol_id", scoped.MaterializationSql, StringComparison.Ordinal); - foreach (var allSymbolsScope in new[] { "fresh", "full", "differential", "retained" }) + foreach (var candidateBoundScope in new[] { "fresh", "full", "differential", "retained" }) { var materialization = Assert.Single( DbWriter.ReferenceResolutionFactSqlForTesting, - entry => entry.Scope == allSymbolsScope).MaterializationSql; + entry => entry.Scope == candidateBoundScope).MaterializationSql; Assert.Contains("FROM symbols AS target", materialization, StringComparison.Ordinal); + Assert.Contains("WHERE EXISTS", materialization, StringComparison.Ordinal); + Assert.Contains( + "INDEXED BY idx_symbol_ref_candidates_symbol", + materialization, + StringComparison.Ordinal); + Assert.Contains( + "candidate.symbol_id = target.id", + materialization, + StringComparison.Ordinal); Assert.DoesNotContain("dirty_target_symbols", materialization, StringComparison.Ordinal); + Assert.DoesNotContain("GROUP BY candidate.symbol_id", materialization, StringComparison.Ordinal); + } + } + + [Fact] + public void ReferenceResolutionFacts_MaterializeOnlyCandidateBearingSymbolsWithReverseSeeks() + { + var callerFileId = InsertFile("src/bounded-caller.py", "python"); + var targetFileId = InsertFile("src/bounded-target.py", "python"); + _writer.InsertSymbols([ + CreateSymbol(targetFileId, "CandidateTarget", line: 1), + CreateSymbol(targetFileId, "UnusedTarget", line: 2), + ]); + _writer.InsertReferences( + [CreateReference(callerFileId, "CandidateTarget", line: 10)], + refreshMutualRecursionFlags: false); + Execute(""" + INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) + SELECT reference.id, target.id, 0 + FROM symbol_references AS reference + JOIN symbols AS target ON target.name = reference.symbol_name; + """); + + Execute(DbWriter.RefreshReferenceResolutionFullSqlForTesting); + + Assert.Equal( + "CandidateTarget", + ScalarString(""" + SELECT target.name + FROM temp.reference_resolution_symbol_facts AS fact + JOIN symbols AS target ON target.id = fact.symbol_id + """)); + Assert.Equal(1, ScalarLong("SELECT COUNT(*) FROM temp.reference_resolution_symbol_facts")); + + var fullFacts = Assert.Single( + DbWriter.ReferenceResolutionFactSqlForTesting, + static entry => entry.Scope == "full").MaterializationSql; + var insert = Assert.Single( + fullFacts.Split( + ';', + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Where(static statement => statement.StartsWith( + "INSERT INTO temp.reference_resolution_symbol_facts", + StringComparison.Ordinal))); + var plan = ReadQueryPlanDetails(insert); + Assert.Contains(plan, static detail => detail.Contains( + "SEARCH candidate USING COVERING INDEX idx_symbol_ref_candidates_symbol", + StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(plan, static detail => + detail.Equals("SCAN candidate", StringComparison.OrdinalIgnoreCase) + || detail.StartsWith("SCAN candidate ", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(plan, static detail => detail.Contains( + "USE TEMP B-TREE", + StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain("DISTINCT", insert, StringComparison.Ordinal); + Assert.DoesNotContain("GROUP BY candidate.symbol_id", insert, StringComparison.Ordinal); + } + + [Fact] + public void SingletonAggregate_MinMaxMatchesCountDistinctNullAndBinarySemantics() + { + Execute(""" + CREATE TEMP TABLE singleton_oracle_values ( + value TEXT COLLATE BINARY + ); + """); + var cases = new (string Name, string?[] Values)[] + { + ("empty", []), + ("all-null", [null, null]), + ("null-and-a", [null, "A"]), + ("duplicate-a", ["A", "A"]), + ("binary-case-variants", ["A", "a"]), + ("a-and-b", ["A", "B"]), + }; + + foreach (var testCase in cases) + { + Execute("DELETE FROM temp.singleton_oracle_values;"); + using (var insert = _db.Connection.CreateCommand()) + { + insert.CommandText = "INSERT INTO temp.singleton_oracle_values(value) VALUES (@value)"; + var value = insert.Parameters.Add("@value", SqliteType.Text); + foreach (var item in testCase.Values) + { + value.Value = item == null ? DBNull.Value : item; + insert.ExecuteNonQuery(); + } + } + + var distinctSingleton = ScalarLong(""" + SELECT COUNT(DISTINCT value COLLATE BINARY) = 1 + FROM temp.singleton_oracle_values + """); + var minMaxSingleton = ScalarLong(""" + SELECT COUNT(value) > 0 + AND MIN(value COLLATE BINARY) IS MAX(value COLLATE BINARY) + FROM temp.singleton_oracle_values + """); + Assert.True( + distinctSingleton == minMaxSingleton, + $"Singleton aggregate mismatch for {testCase.Name}."); } } @@ -861,6 +980,17 @@ private long ScalarLong(string sql) : Convert.ToString(value, CultureInfo.InvariantCulture); } + private IReadOnlyList ReadQueryPlanDetails(string sql) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = "EXPLAIN QUERY PLAN " + sql; + using var reader = command.ExecuteReader(); + var details = new List(); + while (reader.Read()) + details.Add(reader.GetString(3)); + return details; + } + private static int CountOccurrences(string text, string value) { var count = 0; diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs index b0563ba2bf..a84e7be471 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs @@ -203,6 +203,7 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza Assert.Equal(1, captured.Count(snapshot => snapshot.Stage == "deferred_graph_prepared")); Assert.Equal(1, captured.Count(snapshot => snapshot.Stage == "candidate_deferred")); Assert.Equal(1, captured.Count(snapshot => snapshot.Stage == "identity_started")); + Assert.Equal(1, captured.Count(snapshot => snapshot.Stage == "candidate_lookup_restored")); Assert.Equal(1, captured.Count(snapshot => snapshot.Stage == "graph_required_restored")); Assert.Equal(1, captured.Count(snapshot => snapshot.Stage == "mutual_started")); Assert.Equal(1, captured.Count(snapshot => snapshot.Stage == "readiness_completed")); @@ -211,7 +212,7 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza Assert.Equal("dropped", captured[0].Stage); Assert.Equal("full_scan_committed", captured[^1].Stage); Assert.Equal( - ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "graph_required_restored", "mutual_started", "readiness_completed", "restored", "full_scan_committed"], + ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "candidate_lookup_restored", "graph_required_restored", "mutual_started", "readiness_completed", "restored", "full_scan_committed"], captured .Where(snapshot => snapshot.Stage is not "insert_reference_lines" and not "insert_references") .Select(snapshot => snapshot.Stage)); @@ -225,8 +226,8 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza coreIndexPhases); Assert.Equal( rebuild - ? ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "graph_required_restored", "mutual_started", "readiness_completed", "restored", "full_scan_committed"] - : ["dropped", "deferred_graph_prepared", "candidate_deferred", "post_load_statistics_started", "post_load_statistics_completed", "identity_started", "graph_required_restored", "mutual_started", "readiness_completed", "restored", "full_scan_committed"], + ? ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "candidate_lookup_restored", "graph_required_restored", "mutual_started", "readiness_completed", "restored", "full_scan_committed"] + : ["dropped", "deferred_graph_prepared", "candidate_deferred", "post_load_statistics_started", "post_load_statistics_completed", "identity_started", "candidate_lookup_restored", "graph_required_restored", "mutual_started", "readiness_completed", "restored", "full_scan_committed"], lifecycle); var initialBulkNames = rebuild @@ -242,12 +243,15 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza allNames, captured.First(snapshot => snapshot.Stage == "deferred_graph_prepared").Names); Assert.All( - captured.Where(snapshot => snapshot.Stage is "candidate_deferred" or "identity_started" or "graph_required_restored" or "mutual_started" or "readiness_completed"), + captured.Where(snapshot => snapshot.Stage is "candidate_deferred" or "identity_started"), snapshot => { Assert.DoesNotContain("idx_symbol_ref_candidates_symbol", snapshot.Names); Assert.Equal(deferredGraphNames, snapshot.Names); }); + Assert.All( + captured.Where(snapshot => snapshot.Stage is "candidate_lookup_restored" or "graph_required_restored" or "mutual_started" or "readiness_completed"), + snapshot => Assert.Equal(allNames, snapshot.Names)); Assert.Equal( allNames, captured.First(snapshot => snapshot.Stage == "restored").Names); @@ -358,7 +362,7 @@ public void Run_FreshFullScan_DirectGraphRefreshesPostLoadPlannerStatistics() Assert.Equal(1, graphRefreshCount); Assert.Equal(0, augmentationGroupingCount); Assert.Equal( - ["dropped", "candidate_deferred", "post_load_statistics_started", "post_load_statistics_completed", "identity_started", "graph_required_restored", "mutual_started", "restored"], + ["dropped", "candidate_deferred", "post_load_statistics_started", "post_load_statistics_completed", "identity_started", "candidate_lookup_restored", "graph_required_restored", "mutual_started", "restored"], lifecycle); } finally @@ -374,6 +378,7 @@ public void Run_FreshFullScan_DirectGraphRefreshesPostLoadPlannerStatistics() [Theory] [InlineData("dropped")] [InlineData("candidate_deferred")] + [InlineData("candidate_lookup_restored")] [InlineData("graph_required_restored")] [InlineData("readiness_completed")] public void Run_FreshFullScan_FailureDuringStagedReferenceIndexLifecycleRollsBackSchema( @@ -407,9 +412,12 @@ public void Run_FreshFullScan_FailureDuringStagedReferenceIndexLifecycleRollsBac Assert.Equal(failurePhase, statePhases[^1]); Assert.NotNull(failureSnapshot); Assert.Equal( - failurePhase == "dropped" - ? GetAuthoritativeFreshInitialBulkPersistenceReferenceIndexNames() - : GetDeferredGraphPreparationReferenceIndexNames(), + failurePhase switch + { + "dropped" => GetAuthoritativeFreshInitialBulkPersistenceReferenceIndexNames(), + "candidate_deferred" => GetDeferredGraphPreparationReferenceIndexNames(), + _ => GetAllReferenceIndexNames(), + }, failureSnapshot!.Names); Assert.True(File.Exists(dbPath)); @@ -522,8 +530,8 @@ public void Run_HighChurnExistingIndex_DefersReferenceIndexesUntilGraphFinalizat scopedUpdate ? "restored" : "full_scan_committed", captured[^1].Stage); string[] expectedLifecycle = scopedUpdate - ? ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "graph_required_restored", "mutual_started", "readiness_committed", "restored"] - : ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "graph_required_restored", "mutual_started", "readiness_completed", "restored", "full_scan_committed"]; + ? ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "candidate_lookup_restored", "graph_required_restored", "mutual_started", "readiness_committed", "restored"] + : ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "candidate_lookup_restored", "graph_required_restored", "mutual_started", "readiness_completed", "restored", "full_scan_committed"]; Assert.Equal( expectedLifecycle, captured @@ -539,12 +547,15 @@ public void Run_HighChurnExistingIndex_DefersReferenceIndexesUntilGraphFinalizat GetAllReferenceIndexNames(), captured.First(snapshot => snapshot.Stage == "deferred_graph_prepared").Names); Assert.All( - captured.Where(snapshot => snapshot.Stage is "candidate_deferred" or "identity_started" or "graph_required_restored" or "mutual_started" or "readiness_completed" or "readiness_committed"), + captured.Where(snapshot => snapshot.Stage is "candidate_deferred" or "identity_started"), snapshot => { Assert.DoesNotContain("idx_symbol_ref_candidates_symbol", snapshot.Names); Assert.Equal(GetDeferredGraphPreparationReferenceIndexNames(), snapshot.Names); }); + Assert.All( + captured.Where(snapshot => snapshot.Stage is "candidate_lookup_restored" or "graph_required_restored" or "mutual_started" or "readiness_completed" or "readiness_committed"), + snapshot => Assert.Equal(GetAllReferenceIndexNames(), snapshot.Names)); Assert.Equal( GetAllReferenceIndexNames(), captured.First(snapshot => snapshot.Stage == "restored").Names); @@ -920,11 +931,11 @@ public void Run_HighChurnScopedUpdate_FailureAfterReadinessCommitRestoresCanonic exception.Message); Assert.True(readinessCommitObserved); Assert.Equal( - ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "graph_required_restored", "mutual_started", "readiness_committed", "restored"], + ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "candidate_lookup_restored", "graph_required_restored", "mutual_started", "readiness_committed", "restored"], phases); Assert.NotNull(failureSnapshot); Assert.Equal( - GetDeferredGraphPreparationReferenceIndexNames(), + GetAllReferenceIndexNames(), failureSnapshot!.Names); Assert.Equal( expectedAugmentationVersion, diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index c7abde5f41..8c027c88d0 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -8145,6 +8145,7 @@ static string[] ReadPresentIndexes( "post_load_statistics_started", "post_load_statistics_completed", "identity_started", + "candidate_lookup_restored", "graph_required_restored", "mutual_started", "readiness_committed", @@ -8163,6 +8164,7 @@ static string[] ReadPresentIndexes( "deferred_graph_prepared", "candidate_deferred", "identity_started", + "candidate_lookup_restored", "graph_required_restored", "mutual_started", "readiness_committed", @@ -8188,12 +8190,16 @@ static string[] ReadPresentIndexes( static state => state.Phase == "deferred_graph_prepared").PresentIndexNames); Assert.All( referenceIndexStates.Where( - static state => state.Phase is "candidate_deferred" or "identity_started" or "graph_required_restored" or "mutual_started" or "readiness_committed"), + static state => state.Phase is "candidate_deferred" or "identity_started"), state => { Assert.DoesNotContain(candidateReverseIndexName, state.PresentIndexNames); Assert.Equal(deferredGraphPreparedIndexNames, state.PresentIndexNames); }); + Assert.All( + referenceIndexStates.Where( + static state => state.Phase is "candidate_lookup_restored" or "graph_required_restored" or "mutual_started" or "readiness_committed"), + state => Assert.Equal(canonicalReferenceIndexNames, state.PresentIndexNames)); Assert.Contains( candidateReverseIndexName, referenceIndexStates[restoredStateIndex].PresentIndexNames); @@ -10323,7 +10329,7 @@ static string SizedSource(char fill, int size) Assert.Equal(1, optimizeCount); Assert.Equal(0, mergeCount); Assert.Equal( - ["dropped", "candidate_deferred", "identity_started", "graph_required_restored", "mutual_started", "restored"], + ["dropped", "candidate_deferred", "identity_started", "candidate_lookup_restored", "graph_required_restored", "mutual_started", "restored"], referenceIndexPhases); Assert.NotNull(graphScopeStats); Assert.True(graphScopeStats!.UsedFullRefresh); diff --git a/tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs b/tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs index 76971de99f..d011e555de 100644 --- a/tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs +++ b/tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs @@ -100,6 +100,7 @@ public void CanonicalSets_PartitionRawGraphAndRemainingIndexesExactly() [Theory] [InlineData("candidate")] + [InlineData("resolution")] [InlineData("deferred_graph")] [InlineData("mutual")] [InlineData("complete")] @@ -122,6 +123,10 @@ public void AuthoritativeFreshPersistenceDeferral_RestoresBeforeEveryReadDepende case "candidate": guard.PrepareForCandidatePopulation(); break; + case "resolution": + guard.PrepareForCandidatePopulation(); + guard.PrepareForReferenceResolution(); + break; case "deferred_graph": guard.PrepareForDeferredGraphRefresh(); break; @@ -155,10 +160,12 @@ public void TransactionalComplete_RestoresGraphSubsetBeforeRemainingCanonicalSet AssertRawPersistenceIndexesPresent(_db.Connection); guard.PrepareForCandidatePopulation(); + guard.PrepareForReferenceResolution(); guard.PrepareForMutualRecursion(); AssertGraphFinalizationIndexesPresent(_db.Connection); - AssertRemainingQueryIndexesAbsent(_db.Connection); + AssertCandidateResolutionIndexPresent(_db.Connection); + AssertRemainingQueryIndexesExceptCandidateAbsent(_db.Connection); guard.Complete(); @@ -199,6 +206,11 @@ public void TransactionalPrepareForDeferredGraphRefresh_KeepsCandidateUntilPopul Assert.Equal(expectedPreparedNames, preparedNames.Order(StringComparer.Ordinal)); Assert.DoesNotContain(candidateReverseIndexName, preparedNames); + guard.PrepareForReferenceResolution(); + + preparedNames = ReadReferenceIndexNames(_db.Connection); + Assert.Contains(candidateReverseIndexName, preparedNames); + guard.Complete(); Assert.Equal( @@ -257,6 +269,37 @@ public void TransactionalCancelledFreshPersistenceRestore_RemainsRetryable() transaction.Commit(); } + [Fact] + public void TransactionalCancelledCandidateLookupRestore_RemainsRetryable() + { + const string candidateReverseIndexName = "idx_symbol_ref_candidates_symbol"; + using var transaction = _writer.BeginTransaction(); + using var guard = ReferenceSecondaryIndexBulkLoadGuard.StartTransactional( + _writer, + enabled: true); + Assert.NotNull(guard); + + guard.PrepareForCandidatePopulation(); + Assert.DoesNotContain( + candidateReverseIndexName, + ReadReferenceIndexNames(_db.Connection)); + + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + Assert.Throws(() => + guard.PrepareForReferenceResolution(cancellation.Token)); + Assert.DoesNotContain( + candidateReverseIndexName, + ReadReferenceIndexNames(_db.Connection)); + + guard.PrepareForReferenceResolution(); + Assert.Contains( + candidateReverseIndexName, + ReadReferenceIndexNames(_db.Connection)); + guard.Complete(); + transaction.Commit(); + } + [Fact] public void BulkLoad_RollsBackRetiredIndexesTransactionallyButNeverRestoresThemRecoverably() { @@ -336,13 +379,16 @@ public void RecoverableCancelledComplete_AfterGraphRestoreRepairsRemainingIndexe { Assert.NotNull(guard); guard.PrepareForCandidatePopulation(); + guard.PrepareForReferenceResolution(); guard.PrepareForMutualRecursion(); AssertGraphFinalizationIndexesPresent(_db.Connection); - AssertRemainingQueryIndexesAbsent(_db.Connection); + AssertCandidateResolutionIndexPresent(_db.Connection); + AssertRemainingQueryIndexesExceptCandidateAbsent(_db.Connection); cancellation.Cancel(); Assert.Throws(() => guard.Complete(cancellation.Token)); - AssertRemainingQueryIndexesAbsent(_db.Connection); + AssertCandidateResolutionIndexPresent(_db.Connection); + AssertRemainingQueryIndexesExceptCandidateAbsent(_db.Connection); } AssertDeferredIndexesPresent(_db.Connection); @@ -434,6 +480,7 @@ public void FreshPlannerStatistics_RunOnceAfterCandidateDropAndAnalyzeOnlyGraphT guard.PrepareForCandidatePopulation(); guard.PrepareForCandidatePopulation(); guard.ReportIdentityRefreshStarted(); + guard.PrepareForReferenceResolution(); guard.Complete(); transaction.Commit(); @@ -446,6 +493,7 @@ public void FreshPlannerStatistics_RunOnceAfterCandidateDropAndAnalyzeOnlyGraphT "post_load_statistics_completed", "candidate_deferred", "identity_started", + "candidate_lookup_restored", "restored", ], lifecycle); @@ -664,7 +712,8 @@ public void StagedRefresh_PreservesMutualChangesCountAcrossRemainingIndexRestore Assert.Equal(2, ReadChanges(_db.Connection)); AssertGraphFinalizationIndexesPresent(_db.Connection); - AssertRemainingQueryIndexesAbsent(_db.Connection); + AssertCandidateResolutionIndexPresent(_db.Connection); + AssertRemainingQueryIndexesExceptCandidateAbsent(_db.Connection); guard.Complete(); @@ -719,9 +768,11 @@ public void InitializeSchema_RepairsRemainingIndexesAfterAbandonedGraphRestore() enabled: true); Assert.NotNull(abandonedGuard); abandonedGuard.PrepareForCandidatePopulation(); + abandonedGuard.PrepareForReferenceResolution(); abandonedGuard.PrepareForMutualRecursion(); AssertGraphFinalizationIndexesPresent(_db.Connection); - AssertRemainingQueryIndexesAbsent(_db.Connection); + AssertCandidateResolutionIndexPresent(_db.Connection); + AssertRemainingQueryIndexesExceptCandidateAbsent(_db.Connection); // Model termination between graph finalization and query-index restoration. _db.Dispose(); @@ -786,6 +837,27 @@ private static void AssertRemainingQueryIndexesAbsent(SqliteConnection connectio Assert.DoesNotContain(definition.Name, names); } + private static void AssertCandidateResolutionIndexPresent(SqliteConnection connection) + { + var names = ReadReferenceIndexNames(connection); + foreach (var definition in ReferenceSecondaryIndexSql.CandidatePopulationDeferred) + Assert.Contains(definition.Name, names); + } + + private static void AssertRemainingQueryIndexesExceptCandidateAbsent( + SqliteConnection connection) + { + var candidateNames = ReferenceSecondaryIndexSql.CandidatePopulationDeferred + .Select(static definition => definition.Name) + .ToHashSet(StringComparer.Ordinal); + var names = ReadReferenceIndexNames(connection); + foreach (var definition in ReferenceSecondaryIndexSql.RemainingQuery) + { + if (!candidateNames.Contains(definition.Name)) + Assert.DoesNotContain(definition.Name, names); + } + } + private static long ReadChanges(SqliteConnection connection) { using var command = connection.CreateCommand(); From 654de63fa6cd452364fc9df35990411e9eed5214 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 30 Aug 2026 02:28:42 +0900 Subject: [PATCH 11/11] Materialize C# instantiation family facts --- DEVELOPER_GUIDE.md | 60 +- TESTING_GUIDE.md | 2 + ...harp-instantiation-family-facts.changed.md | 14 + .../DbWriter.ReferenceGraphRefreshScope.cs | 24 +- src/CodeIndex/Database/DbWriter.References.cs | 728 +++++++++--------- tests/CodeIndex.Tests/DatabaseTests.cs | 530 ++++++++++++- 6 files changed, 905 insertions(+), 453 deletions(-) create mode 100644 changelog.d/unreleased/+initial-full-index-csharp-instantiation-family-facts.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 5cc8308393..82b9399194 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -473,12 +473,13 @@ restore the remainder after that update. Small scoped updates must keep every index in place so a fixed rebuild cost does not dominate the update. C# reference-graph finalization materializes reference arity, invocation arity, -member-receiver, definition arity, constructor arity, and value-type facts once -per applicable row in TEMP tables. Full, scoped, and retained-graph rebuilds must -then materialize project/file-local type identities and constructor-owner identity -and arity facts from those symbol facts. Before property-receiver normalization, -also materialize C# field/property target identities into a primary-keyed TEMP -fact set. Populate all fact sets before +member-receiver, definition arity, constructor arity, constructor binding sensitivity, +and value-type facts once per applicable row in TEMP tables. Full, scoped, and +retained-graph rebuilds must then materialize project/file-local type identities, +constructor-owner identity and arity facts, and one primary-keyed instantiation-family +fact row for every eligible type declaration and constructor. Before property-receiver +normalization, also materialize C# field/property target identities into a primary-keyed +TEMP fact set. Populate all fact sets before property-receiver normalization, candidate construction, and resolution. Keep candidate SQL on primary-key fact lookups instead of rebuilding identity strings, rescanning constructor-owner ranges, or re-entering managed SQLite scalar functions @@ -487,6 +488,11 @@ lookup-name set and derive identity facts from that bounded population; full and retained rebuilds use the complete C# symbol-fact population. Property-receiver normalization must likewise drive from flagged reference facts and the target fact primary key; scoped target materialization is restricted to its lookup-name set. +Instantiation-family materialization must drive from the already-bounded type and +constructor identity facts into persistent symbols by primary-key seek. Ranks 0–4 +join those facts by symbol ID. The lower-rank binding-sensitive flag includes every +partial type declaration and constructor in an identity, while the rank-5 flag +includes constructors plus only the deterministic representative type declaration. Language-independent scope ranks 1–4 must build their shared reference/name/language candidate relation once in a materialized CTE, assign each reference/symbol pair its best @@ -502,12 +508,17 @@ ambiguity contracts remain unchanged. Scoped refreshes must build the set by driving from dirty reference IDs into the candidate primary key, and every graph pass must clear it before materialization so retries cannot observe stale rows. -The unqualified C# rank-5 type fallback materializes physical type members from -the shared symbol and type-identity facts, groups them into unique logical -families by exact name, arity, and identity, and matches each reference to that -family once. Only the final projection expands a matched family back to every -physical member. This preserves row-per-symbol candidates for partial types while -avoiding repeated compatibility and ambiguity work for every partial declaration. +The unqualified C# rank-5 instantiation fallback consumes the shared family facts +instead of rebuilding type and constructor families per reference. Its uniqueness +flag is computed from type declarations only, grouped by folded name, exact BINARY +name, and arity with non-NULL-count plus BINARY min/max identity equality; a row is +eligible only when its identity is that unique type identity. Constructor-only +orphans therefore remain available to lower ranks but never create a global family. +The reference-driven query uses the explicit composite family-fact index and emits +constructors plus only the deterministic representative type. This preserves +project/file-local conflicts, partial rows, overload/default/optional/`params`, +value-type, enum, delegate, unknown-arity, and ambiguity semantics without a +correlated persistent-symbol scan. Resolution also materializes the nullable target-family key once per candidate-bearing target symbol into a primary-keyed TEMP fact table. Full, fresh, differential, and @@ -2800,7 +2811,7 @@ Process exit codes are coarse (`0` success including valid zero-row queries, `1` - **Dependency-cycle cursors bind presentation evidence as well as topology** — the graph fingerprint includes each retained evidence row's source language, origin, resolution state, reference kind, target kind, suppression reason, and count in deterministic order. A metadata-only graph refresh therefore rejects an older cursor instead of mixing evidence summaries from different snapshots. MCP `format=json-graph` cycle requests use the same bounded node/edge projection as CLI graph output, and the specialized graph node/edge schema limits match the maximum cycle graph budget (#5197). - **No ORM** — Raw `Microsoft.Data.Sqlite` with parameterized queries. Keeps dependencies minimal and control explicit. - **Batch commits** — 500 records per transaction for write performance. Reduces fsync overhead. -- **Set-based C# instantiation fallback** — The rank-5 unqualified `instantiate` candidate stage materializes C# type members, unique raw-name/arity families, family-scoped constructor members, and per-family explicit-constructor summaries before matching references. Unique families drive indexed constructor lookups instead of scanning every constructor or running correlated type/constructor scalar probes per candidate. Raw type names, identities, and constructor containers remain `BINARY`, family/member arity joins remain NULL-safe, partial types keep their path/start/id representative, and the final lower-rank suppression stays reference-scoped so the optimization preserves overload, implicit-default, value-type, enum, delegate, ambiguity, and unparseable-arity semantics. +- **Fact-backed C# instantiation families** — Graph finalization materializes one TEMP row per eligible C# type declaration and constructor after identity facts are ready. Ranks 0–4 use symbol-primary-key joins and rank 5 uses a reference-driven composite family seek, so no candidate path performs a correlated constructor-family symbol scan. Separate lower-rank and fallback binding-sensitive flags preserve all-partial versus representative-only primary-constructor semantics. Type-declaration-only BINARY uniqueness keeps project/file-local conflicts ambiguous and constructor-only orphans out of rank 5 while retaining overload, implicit-default, optional/default/`params`, value-type, enum, delegate, and unknown-arity behavior. - **Partial batch failures** — `DbWriter` keeps the fast multi-row `INSERT` path for normal chunk and symbol batches. If SQLite rejects a batch, the writer rolls that batch back, retries rows under per-row `SAVEPOINT`s, commits the valid rows, skips only the failing rows, increments `BatchRowsSkipped`, and emits a warning containing the row identifier and SQLite error. This keeps one corrupt extracted row from discarding the rest of a large indexing batch (#1754). - **WAL mode + busy_timeout** — Write-Ahead Logging for concurrent read/write access and crash safety. 5-second busy timeout avoids immediate SQLITE_BUSY errors. - **Content-external FTS5 with triggers** — Avoids doubling storage by pointing to `chunks` table instead of storing a copy. Database triggers keep the FTS index in sync automatically. @@ -4721,8 +4732,9 @@ full mutual-recursion update は、call-like または非canonicalな row ごと single-evaluation の契約を維持してください。 C# の reference-graph finalization は、reference arity、invocation arity、member receiver、 -definition arity、constructor arity、value-type の fact を、対象 row ごとに TEMP table へ1回だけ -materialize し、その symbol fact から project / file-local type identity と constructor-owner の identity / arity +definition arity、constructor arity、constructor binding sensitivity、value-type の fact を、対象 row ごとに +TEMP table へ1回だけ materialize し、その symbol fact から project / file-local type identity、constructor-owner +の identity / arity、および対象となる全 type declaration / constructor の primary-keyed instantiation-family fact も materialize します。property-receiver normalization の前に C# field / property の target identity も primary-keyed TEMP fact 集合へ materialize します。full / scoped / retained graph rebuild の全経路で fact 集合を property-receiver normalization、candidate 構築、resolution より前に投入してください。candidate SQL は @@ -4731,6 +4743,10 @@ scalar function へ再入したりせず、primary-key の fact lookup を使い lookup-name 集合だけに限定し、identity fact もその限定済み集合から作ります。full / retained rebuild は C# symbol fact の全対象を使います。property-receiver normalization も flag 済み reference fact と target fact の primary key から駆動し、scoped target materialization は lookup-name 集合だけに限定してください。 +instantiation-family materialization は限定済み type / constructor identity fact を外側にして、永続 symbol を +primary key で seek します。rank 0〜4 は symbol ID でこの fact を join します。lower-rank の binding-sensitive +flag は同一 identity の全 partial type declaration と constructor を含め、rank 5 用 flag は constructor と +決定的な代表 type declaration だけを含めてください。 言語共通の scope rank 1〜4 は、共有する reference / name / language candidate relation を materialized CTE で1回だけ構築し、reference / symbol pair ごとの最良rankを割り当てたうえで、 @@ -4745,11 +4761,13 @@ ambiguity 契約は変更しません。scoped refresh は dirty reference ID seek して集合を作り、retry が古い行を参照しないよう graph pass ごとに materialize 前の clear を 維持してください。 -qualifier のない C# rank 5 type fallback は、共有 symbol / type-identity fact から物理 type member を -materializeし、exact name・arity・identityごとの一意な論理familyへgroup化して、referenceごとの照合を -family単位で1回だけ行います。一致したfamilyを全物理memberへ展開するのは最終projectionだけです。 -これによりpartial typeのsymbolごとのcandidate行を維持しつつ、各partial宣言でcompatibilityとambiguity -判定を繰り返しません。 +qualifier のない C# rank 5 instantiation fallback は、reference ごとに type / constructor family を再構築せず、 +共有 family fact を使います。一意性 flag の母集団は type declaration だけで、fold済みname、BINARY exact name、 +arity ごとに非NULL件数とBINARY identityのmin/max一致を判定し、row自身のidentityがその一意identityに一致する +場合だけ有効にします。そのため constructor しかない orphan はlower rankでは候補になれてもglobal familyを +作りません。reference側から明示的なcomposite family-fact indexをseekし、constructorと決定的な代表typeだけを +出力します。project / file-local競合、partial row、overload、default / optional / `params`、value type、enum、 +delegate、arity不明、ambiguityのsemanticsを保ちつつ、相関した永続symbol scanを行いません。 resolution は nullable な target-family key も candidate を持つ target symbol ごとに1回だけ primary-keyed TEMP fact table へ materialize します。full / fresh / differential / retained refresh は @@ -7002,7 +7020,7 @@ USER_GUIDEの[終了コード](USER_GUIDE.md#終了コード)セクションを - **依存 cycle cursor は topology に加えて表示 evidence にも束縛する** — graph fingerprint は retained evidence 各行の source language、origin、resolution state、reference kind、target kind、suppression reason、件数を決定的な順序で含めます。そのため metadata だけが更新された graph でも古い cursor を拒否し、異なる snapshot の evidence summary を混在させません。MCP の `format=json-graph` cycle request は CLI graph 出力と同じ上限付き node / edge 投影を使い、専用 graph node / edge schema の上限も cycle graph budget の最大値と一致させます（#5197）。 - **ORMなし** — `Microsoft.Data.Sqlite`でパラメータ化クエリを直接使用。依存関係を最小限に、制御を明確に。 - **バッチコミット** — 書き込み性能のため1トランザクション500レコード。fsyncオーバーヘッドを削減。 -- **C# instantiation fallback の集合処理** — rank 5 の無修飾 `instantiate` candidate 段階は、参照との照合前に C# type member、raw name / arity 単位の一意 family、family 内 constructor member、family ごとの明示 constructor summary を materialize します。一意 family から indexed constructor lookup を駆動するため、全 constructor scan や candidate ごとの相関 type / constructor scalar probe を行いません。raw type name・identity・constructor container は `BINARY`、family/member の arity join は NULL-safe のまま維持し、partial type は path/start/id 順の代表を使い、最後の lower-rank suppression も reference 単位に保つため、overload、implicit default、value type、enum、delegate、ambiguity、arity を解析できない場合の意味を変えずに高速化します。 +- **fact-backed C# instantiation family** — graph finalization はidentity fact完成後、対象となるC# type declaration / constructorごとにTEMP rowを1件materializeします。rank 0〜4はsymbol primary key join、rank 5はreference側からcomposite family indexをseekするため、candidate経路で相関constructor-family symbol scanを行いません。lower-rankとfallbackのbinding-sensitive flagを分離し、全partialと代表typeだけのprimary-constructor semanticsを保ちます。type declarationだけを母集団にしたBINARY一意性によりproject / file-local競合はambiguousのまま、constructor-only orphanはrank 5から除外し、overload、implicit default、optional / default / `params`、value type、enum、delegate、arity不明の挙動を維持します。 - **部分的なバッチ失敗** — `DbWriter` は通常の chunk / symbol batch では高速な multi-row `INSERT` 経路を保ちます。SQLite が batch を拒否した場合、その batch を rollback し、各 row を per-row `SAVEPOINT` の下で再試行し、有効な row だけを commit し、失敗 row だけを skip して `BatchRowsSkipped` を増やし、row identifier と SQLite error を含む warning を出します。これにより、抽出された 1 行の破損で大きな indexing batch 全体が捨てられることを防ぎます（#1754）。 - **WALモード + busy_timeout** — Write-Ahead Loggingで読み書き同時アクセスとクラッシュ安全性を確保。5秒のbusy_timeoutで即座のSQLITE_BUSYエラーを回避。 - **複数 SELECT をまたぐ reader の snapshot 隔離** — 1 回の呼び出しで複数 SQL を発行する read エントリポイント（`DbReader.GetStatus`、`DbReader.AnalyzeSymbol`（CLI `inspect` / MCP `analyze_symbol`）、`RepoMapBuilder.Build`（CLI `map` / MCP `repo_map`））は、本体を 1 つの `BEGIN DEFERRED` transaction で囲み、すべての sub-query が同じ WAL snapshot を参照するようにする。これが無いと、2 つの `COUNT(*)` の間に writer が commit した結果として並行 reader が `files=836, refs=0` のような不整合状態を観測しうる（issue #180 で露見）。`DEFERRED` は最初の SELECT で `SHARED` lock を取るだけで writer を阻害せず、末尾で明示 Commit して `SHARED` lock を早期解放する。独自に `SqliteDataReader` を開く sub-query は内側ブロックに閉じ込めて `Commit()` より前に handle を解放すること — `SqliteTransaction.Commit()` は同じ connection 上で開いている reader があると失敗する。新しい多段 read エントリポイントは同じパターンに従うこと。単一 SQL のクエリは SQLite の auto-commit が文単位の snapshot を与えるため不要。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index cfd3a7424a..88dcaf2013 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -818,6 +818,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` also fixes language-independent ranks 1–4 at one materialized candidate relation plus one minimum-rank relation, while scoped SQL must drive that relation from dirty IDs into reference primary keys. It keeps the rank-5 lower-rank guard at one compact TEMP row per matched reference, and each of the five rank-5 fallbacks must use a TEMP primary-key seek. `ReferenceScopeCandidates_MaterializeMinimumRankAndPreserveTies` proves C# and Python retain rank 1–4 precedence, every minimum-rank tie, the source-less same-file fallback, and identical scoped/full/retained results. The C# type fallback must materialize physical members, unique logical families, and matched families in that order before its final physical expansion; retain primary-key seeks for scoped symbols, references, and facts. `CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember` proves that same-identity partial declarations still produce every physical candidate while a conflicting identity suppresses the whole rank-5 family across scoped, full, and retained refreshes. Keep the remaining full/scoped/retained resolution oracles beside these structural guards so physical candidate rows and multi-language ambiguity remain unchanged. + `DatabaseTests.CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers` additionally fixes C# instantiation-family construction to type/constructor identity TEMP scans followed by persistent-symbol and symbol-fact primary-key seeks. Rank 5 must drive from full/retained references or scoped dirty-reference primary keys into the explicit `(name_folded, name, fallback_family_is_unique, ...)` index; reject automatic indexes and correlated constructor-symbol probes. Pair that plan guard with `CSharpInstantiationFallback_SetBasedFamiliesPreserveSemanticBoundaries`, `CSharpInstantiationFamilies_KeepLowerRankAndFallbackPartialSensitivitySeparate`, and the graph-identity rollback/cancellation tests. Together they pin optional/default/`params`, partial representatives, type-only BINARY uniqueness, same-name identity conflicts, constructor-only orphan behavior, and transactional full/scoped/retained retries. `DatabaseTests.CSharpPropertyReceiverNormalization_SeeksFactBackedReferencesAndTargets` requires both normalization updates to seek flagged reference IDs and the primary-keyed field/property target facts rather than scan all references or persistent target symbols. Keep its full/scoped/retained stage-order assertions and property-resolution fixtures paired so lookup-name scoping cannot change inherited-member semantics. `FreshReferenceResolutionTests.ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope` fixes target-family key construction to one per target symbol and candidate resolution to TEMP primary-key facts across fresh, full, differential, scoped, and retained paths. Its legacy-null-key and C#/Python oracle coverage preserves resolved IDs, exact keys, grouped families, ambiguity, and self-reference semantics. - `HotspotReferenceAggregateTests.cs` @@ -1969,6 +1970,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` は、言語共通のrank 1〜4を1つのmaterialized candidate relationと1つのminimum-rank relationへ固定し、scoped SQLがdirty IDからreference primary keyへ駆動することも要求します。rank 5の下位rank guardは、一致したreferenceごとに1行のcompact TEMP集合を維持し、5つのrank 5 fallbackはそれぞれTEMP primary-key seekを使う必要があります。`ReferenceScopeCandidates_MaterializeMinimumRankAndPreserveTies`はC# / Pythonでrank 1〜4の優先順位、最小rankの全同順位、source不明時のsame-file fallback、scoped / full / retainedの同一結果を証明します。C# type fallbackは最終の物理展開より前に、物理member、一意な論理family、一致familyの順でmaterializeし、scoped symbol / reference / factのprimary-key seekを維持してください。`CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember`は、同一identityのpartial宣言が引き続き全物理candidateを生成し、競合identityがscoped / full / retained refreshを横断してrank 5 family全体を抑止することを証明します。物理candidate行と多言語ambiguityが変わらないよう、残りのfull / scoped / retained resolution oracleもこれらの構造guardと対で維持してください。 + `DatabaseTests.CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers` はさらに、C# instantiation-family構築をtype / constructor identity TEMP scanから永続symbolとsymbol factのprimary-key seekへ固定します。rank 5はfull / retained referenceまたはscoped dirty-reference primary keyから明示的な`(name_folded, name, fallback_family_is_unique, ...)` indexをseekし、automatic indexと相関constructor-symbol probeを拒否します。このplan guardは`CSharpInstantiationFallback_SetBasedFamiliesPreserveSemanticBoundaries`、`CSharpInstantiationFamilies_KeepLowerRankAndFallbackPartialSensitivitySeparate`、graph-identityのrollback / cancellation testと対で維持してください。これらによりoptional / default / `params`、partial代表、type-only BINARY一意性、同名identity競合、constructor-only orphan、transactionalなfull / scoped / retained retryを固定します。 `DatabaseTests.CSharpPropertyReceiverNormalization_SeeksFactBackedReferencesAndTargets` は、2つのnormalization updateが全referenceや永続target symbolをscanせず、flag済みreference IDとprimary-keyed field / property target factをseekすることを要求します。lookup-name scopeが継承member semanticsを変えないよう、full / scoped / retainedのstage-order assertionとproperty-resolution fixtureを対で維持してください。 `FreshReferenceResolutionTests.ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope` は、target-family key構築をtarget symbolごと1回に限定し、fresh / full / differential / scoped / retainedのcandidate resolutionがTEMP primary-key factを使う契約を固定します。legacy null-keyとC# / Python oracle coverageにより、resolved ID、exact key、group family、ambiguity、self-reference semanticsを維持します。 - `HotspotReferenceAggregateTests.cs` diff --git a/changelog.d/unreleased/+initial-full-index-csharp-instantiation-family-facts.changed.md b/changelog.d/unreleased/+initial-full-index-csharp-instantiation-family-facts.changed.md new file mode 100644 index 0000000000..eddd6f39d6 --- /dev/null +++ b/changelog.d/unreleased/+initial-full-index-csharp-instantiation-family-facts.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs + - src/CodeIndex/Database/DbWriter.References.cs +--- + +## English + +- **Cold C# constructor matching reuses family facts** — Reference-graph finalization now materializes constructor-family compatibility once per relevant type or constructor and reuses indexed TEMP facts across every candidate rank. This removes per-candidate correlated symbol scans while preserving partial, project/file-local, overload, optional/default/`params`, orphan, value-type, and legacy ambiguity semantics. + +## 日本語 + +- **初回 C# constructor 照合で family fact を再利用** — reference-graph finalization は対象type / constructorごとにconstructor-family互換性を1回だけmaterializeし、全candidate rankでindexed TEMP factを再利用します。candidateごとの相関symbol scanをなくしつつ、partial、project / file-local、overload、optional / default / `params`、orphan、value type、legacy ambiguityのsemanticsを維持します。 diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index 1be16420f6..bc9f8de87d 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -206,6 +206,7 @@ internal static string RefreshScopedReferenceCandidatesSqlForTesting + RefreshCSharpSymbolFactsFullSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpInstantiationFamilyFactsSql + "\n" + RefreshCSharpPropertyTargetFactsFullSql + "\n" + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceCandidatesSql), @@ -215,6 +216,7 @@ internal static string RefreshScopedReferenceCandidatesSqlForTesting + RefreshCSharpSymbolFactsScopedSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpInstantiationFamilyFactsSql + "\n" + RefreshCSharpPropertyTargetFactsScopedSql + "\n" + NormalizeCSharpPropertyReceiverReferencesScopedSql + "\n" + RefreshScopedReferenceCandidatesSql), @@ -224,6 +226,7 @@ internal static string RefreshScopedReferenceCandidatesSqlForTesting + RefreshCSharpSymbolFactsFullSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpInstantiationFamilyFactsSql + "\n" + RefreshCSharpPropertyTargetFactsFullSql + "\n" + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceCandidatesSql), @@ -238,6 +241,15 @@ internal static string RefreshScopedReferenceCandidatesSqlForTesting ("retained", RefreshReferenceCandidatesSql), ]; + internal static IReadOnlyList<(string Scope, string Sql)> + CSharpInstantiationFamilyFactSqlForTesting + => + [ + ("full", RefreshCSharpInstantiationFamilyFactsSql), + ("scoped", RefreshCSharpInstantiationFamilyFactsSql), + ("retained", RefreshCSharpInstantiationFamilyFactsSql), + ]; + internal static IReadOnlyList<( string Scope, string MaterializationSql, @@ -298,8 +310,6 @@ private static string BuildScopedReferenceCandidatesSql() { const string fullDeleteSql = "DELETE FROM symbol_reference_candidates;"; const string fullReferenceSourceSql = "FROM symbol_references AS r"; - const string fullInstantiateSymbolSourceSql = "FROM symbols AS s"; - const string fullInstantiateNamePredicateSql = "AND s.name_folded IS NOT NULL"; const string fullCSharpTypeSymbolSourceSql = "FROM symbols AS type_symbol"; const string fullCSharpTypeNamePredicateSql = "AND type_symbol.name_folded IS NOT NULL"; const string fullLowerRankCandidateSourceSql = @@ -309,8 +319,6 @@ private static string BuildScopedReferenceCandidatesSql() if (CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullDeleteSql) != 1 || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullReferenceSourceSql) != expectedReferenceSourceCount - || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullInstantiateSymbolSourceSql) != 1 - || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullInstantiateNamePredicateSql) != 1 || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullCSharpTypeSymbolSourceSql) != 1 || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullCSharpTypeNamePredicateSql) != 1 || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullLowerRankCandidateSourceSql) != 1) @@ -328,14 +336,6 @@ private static string BuildScopedReferenceCandidatesSql() fullReferenceSourceSql, $"FROM temp.{ReferenceGraphDirtyReferencesTable} AS dirty_reference\n CROSS JOIN symbol_references AS r ON r.id = dirty_reference.reference_id", StringComparison.Ordinal) - .Replace( - fullInstantiateSymbolSourceSql, - $"FROM temp.{ReferenceGraphLookupNamesTable} AS lookup_name\n CROSS JOIN symbols AS s INDEXED BY idx_symbols_name_folded", - StringComparison.Ordinal) - .Replace( - fullInstantiateNamePredicateSql, - "AND lookup_name.lang = 'csharp'\n AND s.name_folded = lookup_name.name_folded", - StringComparison.Ordinal) .Replace( fullCSharpTypeSymbolSourceSql, $"FROM temp.{ReferenceGraphLookupNamesTable} AS type_lookup_name\n CROSS JOIN symbols AS type_symbol INDEXED BY idx_symbols_name_folded", diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 763cd041dc..428fe542cf 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -197,11 +197,12 @@ PRIMARY KEY(name_folded, name, container_qualified_name, symbol_id) ) WITHOUT ROWID; CREATE TEMP TABLE IF NOT EXISTS csharp_symbol_facts ( - symbol_id INTEGER NOT NULL PRIMARY KEY, - definition_type_arity INTEGER, - constructor_parameter_count INTEGER, - callable_parameter_count INTEGER, - is_value_type INTEGER NOT NULL + symbol_id INTEGER NOT NULL PRIMARY KEY, + definition_type_arity INTEGER, + constructor_parameter_count INTEGER, + constructor_binding_sensitive INTEGER NOT NULL, + callable_parameter_count INTEGER, + is_value_type INTEGER NOT NULL ) WITHOUT ROWID; CREATE TEMP TABLE IF NOT EXISTS csharp_type_identity_facts ( @@ -216,6 +217,24 @@ CREATE TEMP TABLE IF NOT EXISTS csharp_constructor_identity_facts ( type_arity INTEGER ) WITHOUT ROWID; + CREATE TEMP TABLE IF NOT EXISTS csharp_instantiation_family_facts ( + symbol_id INTEGER NOT NULL PRIMARY KEY, + name_folded TEXT NOT NULL, + name TEXT NOT NULL COLLATE BINARY, + type_arity INTEGER, + type_identity TEXT COLLATE BINARY, + candidate_kind TEXT NOT NULL, + constructor_parameter_count INTEGER, + constructor_binding_sensitive INTEGER NOT NULL, + lower_rank_family_binding_sensitive INTEGER NOT NULL, + fallback_family_binding_sensitive INTEGER NOT NULL, + fallback_family_is_unique INTEGER NOT NULL, + is_value_type INTEGER NOT NULL, + is_representative INTEGER NOT NULL, + has_explicit_constructor INTEGER NOT NULL, + has_explicit_zero_constructor INTEGER NOT NULL + ) WITHOUT ROWID; + {CreateReferenceResolutionSymbolFactsTableSql}; CREATE TEMP TABLE IF NOT EXISTS {ReferenceLowerRankCandidateMatchesTable} ( @@ -226,7 +245,18 @@ reference_id INTEGER NOT NULL PRIMARY KEY private const string CreateCSharpReferenceFactIndexesSql = """ CREATE INDEX IF NOT EXISTS temp.idx_csharp_reference_facts_property_receiver ON csharp_reference_facts(reference_id) - WHERE is_property_receiver_reference = 1 + WHERE is_property_receiver_reference = 1; + + CREATE INDEX IF NOT EXISTS temp.idx_csharp_instantiation_family_facts_lookup + ON csharp_instantiation_family_facts( + name_folded, + name, + fallback_family_is_unique, + type_arity, + type_identity, + is_representative, + candidate_kind, + symbol_id) """; private static readonly string RefreshReferenceResolutionSymbolFactsFullSql = $""" @@ -362,6 +392,7 @@ INSERT INTO temp.csharp_symbol_facts( symbol_id, definition_type_arity, constructor_parameter_count, + constructor_binding_sensitive, callable_parameter_count, is_value_type) SELECT symbol.id, @@ -373,6 +404,10 @@ INSERT INTO temp.csharp_symbol_facts( symbol.signature, symbol.name, symbol.kind), + csharp_constructor_has_binding_sensitive_parameters( + symbol.signature, + symbol.name, + symbol.kind), csharp_callable_parameter_count( symbol.signature, symbol.name, @@ -407,23 +442,6 @@ FROM temp.csharp_symbol_facts AS symbol_fact ) """; - private static string BuildCSharpConstructorParameterCountSql(string symbolAlias) - => $""" - ( - SELECT symbol_fact.constructor_parameter_count - FROM temp.csharp_symbol_facts AS symbol_fact - WHERE symbol_fact.symbol_id = {symbolAlias}.id - ) - """; - - private static string BuildCSharpConstructorBindingSensitiveSql(string symbolAlias) - => $""" - csharp_constructor_has_binding_sensitive_parameters( - {symbolAlias}.signature, - {symbolAlias}.name, - {symbolAlias}.kind) - """; - private static string BuildCSharpCallableParameterCountSql(string symbolAlias) => $""" ( @@ -433,15 +451,6 @@ FROM temp.csharp_symbol_facts AS symbol_fact ) """; - private static string BuildCSharpIsValueTypeSql(string symbolAlias) - => $""" - ( - SELECT symbol_fact.is_value_type - FROM temp.csharp_symbol_facts AS symbol_fact - WHERE symbol_fact.symbol_id = {symbolAlias}.id - ) - """; - private static string BuildCSharpProjectPrefixSql(string symbolAlias) => $""" CASE @@ -586,7 +595,7 @@ JOIN temp.csharp_symbol_facts AS constructor_type_fact WHERE constructor.kind = 'function' AND ( constructor_fact.constructor_parameter_count IS NOT NULL - OR {BuildCSharpConstructorBindingSensitiveSql("constructor")} = 1 + OR constructor_fact.constructor_binding_sensitive = 1 ) ) INSERT INTO temp.csharp_constructor_identity_facts( @@ -622,7 +631,7 @@ JOIN files AS constructor_file WHERE constructor.kind = 'function' AND ( constructor_fact.constructor_parameter_count IS NOT NULL - OR {BuildCSharpConstructorBindingSensitiveSql("constructor")} = 1 + OR constructor_fact.constructor_binding_sensitive = 1 ) AND NOT EXISTS ( SELECT 1 @@ -631,74 +640,251 @@ FROM temp.csharp_constructor_identity_facts AS existing ); """; - private static string BuildCSharpTypeIdentitySql(string symbolAlias) - => $""" - ( - SELECT type_identity_fact.type_identity - FROM temp.csharp_type_identity_facts AS type_identity_fact - WHERE type_identity_fact.symbol_id = {symbolAlias}.id - ) - """; + private const string RefreshCSharpInstantiationFamilyFactsSql = """ + DELETE FROM temp.csharp_instantiation_family_facts; - private static string BuildCSharpConstructorIdentitySql(string symbolAlias) - => $""" - ( - SELECT constructor_identity_fact.type_identity - FROM temp.csharp_constructor_identity_facts AS constructor_identity_fact - WHERE constructor_identity_fact.symbol_id = {symbolAlias}.id - ) - """; - - private static string BuildCSharpConstructorFamilyIdentitySql(string symbolAlias) - => $""" - COALESCE( - {BuildCSharpConstructorIdentitySql(symbolAlias)}, - {BuildCSharpTypeIdentitySql(symbolAlias)}) - """; + WITH csharp_instantiation_type_members( + symbol_id, + name_folded, + name, + type_arity, + type_identity, + candidate_kind, + constructor_parameter_count, + constructor_binding_sensitive, + is_value_type, + representative_rank) AS MATERIALIZED ( + SELECT symbol.id, + symbol.name_folded, + symbol.name COLLATE BINARY, + symbol_fact.definition_type_arity, + type_identity_fact.type_identity COLLATE BINARY, + symbol.kind, + symbol_fact.constructor_parameter_count, + symbol_fact.constructor_binding_sensitive, + symbol_fact.is_value_type, + ROW_NUMBER() OVER ( + PARTITION BY symbol.name_folded, + symbol.name COLLATE BINARY, + symbol_fact.definition_type_arity, + type_identity_fact.type_identity COLLATE BINARY + ORDER BY symbol_file.path COLLATE BINARY, + COALESCE(symbol.start_line, symbol.line), + symbol.id) + FROM temp.csharp_type_identity_facts AS type_identity_fact + CROSS JOIN symbols AS symbol ON symbol.id = type_identity_fact.symbol_id + JOIN files AS symbol_file + ON symbol_file.id = symbol.file_id + AND symbol_file.lang = 'csharp' + JOIN temp.csharp_symbol_facts AS symbol_fact + ON symbol_fact.symbol_id = type_identity_fact.symbol_id + WHERE symbol.name_folded IS NOT NULL + AND symbol.name IS NOT NULL + AND symbol.kind IN ('class', 'struct', 'record', 'enum', 'delegate') + ), + csharp_instantiation_constructor_members( + symbol_id, + name_folded, + name, + type_arity, + type_identity, + candidate_kind, + constructor_parameter_count, + constructor_binding_sensitive, + is_value_type, + representative_rank) AS MATERIALIZED ( + SELECT constructor.id, + constructor.name_folded, + constructor.name COLLATE BINARY, + constructor_identity.type_arity, + constructor_identity.type_identity COLLATE BINARY, + constructor.kind, + constructor_fact.constructor_parameter_count, + constructor_fact.constructor_binding_sensitive, + 0, + 0 + FROM temp.csharp_constructor_identity_facts AS constructor_identity + CROSS JOIN symbols AS constructor + ON constructor.id = constructor_identity.symbol_id + JOIN files AS constructor_file + ON constructor_file.id = constructor.file_id + AND constructor_file.lang = 'csharp' + JOIN temp.csharp_symbol_facts AS constructor_fact + ON constructor_fact.symbol_id = constructor_identity.symbol_id + WHERE constructor.name_folded IS NOT NULL + AND constructor.name IS NOT NULL + AND constructor.kind = 'function' + AND constructor.container_name = constructor.name COLLATE BINARY + AND ( + constructor_fact.constructor_parameter_count IS NOT NULL + OR constructor_fact.constructor_binding_sensitive = 1 + ) + ), + csharp_instantiation_members( + symbol_id, + name_folded, + name, + type_arity, + type_identity, + candidate_kind, + constructor_parameter_count, + constructor_binding_sensitive, + is_value_type, + is_representative) AS MATERIALIZED ( + SELECT type_member.symbol_id, + type_member.name_folded, + type_member.name, + type_member.type_arity, + type_member.type_identity, + type_member.candidate_kind, + type_member.constructor_parameter_count, + type_member.constructor_binding_sensitive, + type_member.is_value_type, + CASE + WHEN type_member.type_identity IS NOT NULL + AND type_member.representative_rank = 1 THEN 1 + ELSE 0 + END + FROM csharp_instantiation_type_members AS type_member - private static string BuildCSharpConstructorFamilyBindingSensitiveSql(string symbolAlias) - => $""" - EXISTS ( - SELECT 1 - FROM symbols AS binding_sensitive_constructor - JOIN files AS binding_sensitive_constructor_file - ON binding_sensitive_constructor_file.id = - binding_sensitive_constructor.file_id - AND binding_sensitive_constructor_file.lang = 'csharp' - LEFT JOIN temp.csharp_constructor_identity_facts AS binding_sensitive_constructor_identity - ON binding_sensitive_constructor_identity.symbol_id = - binding_sensitive_constructor.id - LEFT JOIN temp.csharp_type_identity_facts AS binding_sensitive_type_identity - ON binding_sensitive_type_identity.symbol_id = - binding_sensitive_constructor.id - WHERE binding_sensitive_constructor.name_folded = - {symbolAlias}.name_folded - AND binding_sensitive_constructor.name = - {symbolAlias}.name COLLATE BINARY - AND COALESCE( - binding_sensitive_constructor_identity.type_identity, - binding_sensitive_type_identity.type_identity) = - {BuildCSharpConstructorFamilyIdentitySql(symbolAlias)} COLLATE BINARY - AND ( - ( - binding_sensitive_constructor.kind = 'function' - AND binding_sensitive_constructor.container_name = - binding_sensitive_constructor.name COLLATE BINARY - ) - OR binding_sensitive_constructor.kind IN ('class', 'struct', 'record') - ) - AND {BuildCSharpConstructorBindingSensitiveSql("binding_sensitive_constructor")} = 1 - ) - """; + UNION ALL - private static string BuildCSharpConstructorTypeAritySql(string symbolAlias) - => $""" - ( - SELECT constructor_identity_fact.type_arity - FROM temp.csharp_constructor_identity_facts AS constructor_identity_fact - WHERE constructor_identity_fact.symbol_id = {symbolAlias}.id - ) - """; + SELECT constructor_member.symbol_id, + constructor_member.name_folded, + constructor_member.name, + constructor_member.type_arity, + constructor_member.type_identity, + constructor_member.candidate_kind, + constructor_member.constructor_parameter_count, + constructor_member.constructor_binding_sensitive, + constructor_member.is_value_type, + 0 + FROM csharp_instantiation_constructor_members AS constructor_member + ) + INSERT INTO temp.csharp_instantiation_family_facts( + symbol_id, + name_folded, + name, + type_arity, + type_identity, + candidate_kind, + constructor_parameter_count, + constructor_binding_sensitive, + lower_rank_family_binding_sensitive, + fallback_family_binding_sensitive, + fallback_family_is_unique, + is_value_type, + is_representative, + has_explicit_constructor, + has_explicit_zero_constructor) + SELECT member.symbol_id, + member.name_folded, + member.name COLLATE BINARY, + member.type_arity, + member.type_identity COLLATE BINARY, + member.candidate_kind, + member.constructor_parameter_count, + member.constructor_binding_sensitive, + CASE + WHEN member.type_identity IS NULL THEN 0 + ELSE MAX(CASE + WHEN member.candidate_kind = 'function' + OR member.candidate_kind IN ('class', 'struct', 'record') + THEN member.constructor_binding_sensitive + ELSE 0 + END) OVER ( + PARTITION BY member.name_folded, + member.name COLLATE BINARY, + member.type_arity, + member.type_identity COLLATE BINARY) + END, + CASE + WHEN member.type_identity IS NULL THEN 0 + ELSE MAX(CASE + WHEN member.candidate_kind = 'function' + OR member.is_representative = 1 + THEN member.constructor_binding_sensitive + ELSE 0 + END) OVER ( + PARTITION BY member.name_folded, + member.name COLLATE BINARY, + member.type_arity, + member.type_identity COLLATE BINARY) + END, + CASE + WHEN COUNT(CASE + WHEN member.candidate_kind IN ( + 'class', + 'struct', + 'record', + 'enum', + 'delegate') + THEN member.type_identity + END) OVER ( + PARTITION BY member.name_folded, + member.name COLLATE BINARY, + member.type_arity) > 0 + AND MIN(CASE + WHEN member.candidate_kind IN ( + 'class', + 'struct', + 'record', + 'enum', + 'delegate') + THEN member.type_identity COLLATE BINARY + END) OVER ( + PARTITION BY member.name_folded, + member.name COLLATE BINARY, + member.type_arity) + IS MAX(CASE + WHEN member.candidate_kind IN ( + 'class', + 'struct', + 'record', + 'enum', + 'delegate') + THEN member.type_identity COLLATE BINARY + END) OVER ( + PARTITION BY member.name_folded, + member.name COLLATE BINARY, + member.type_arity) + AND member.type_identity COLLATE BINARY IS MIN(CASE + WHEN member.candidate_kind IN ( + 'class', + 'struct', + 'record', + 'enum', + 'delegate') + THEN member.type_identity COLLATE BINARY + END) OVER ( + PARTITION BY member.name_folded, + member.name COLLATE BINARY, + member.type_arity) + THEN 1 + ELSE 0 + END, + member.is_value_type, + member.is_representative, + CASE + WHEN member.type_identity IS NULL THEN 0 + ELSE MAX(member.candidate_kind = 'function') OVER ( + PARTITION BY member.name_folded, + member.name COLLATE BINARY, + member.type_arity, + member.type_identity COLLATE BINARY) + END, + CASE + WHEN member.type_identity IS NULL THEN 0 + ELSE MAX( + member.candidate_kind = 'function' + AND member.constructor_parameter_count = 0) OVER ( + PARTITION BY member.name_folded, + member.name COLLATE BINARY, + member.type_arity, + member.type_identity COLLATE BINARY) + END + FROM csharp_instantiation_members AS member; + """; private static string CSharpReferenceTypeAritySql => """ ( @@ -783,127 +969,79 @@ OR CASE ) THEN 1 WHEN r.reference_kind = 'call' THEN 0 WHEN r.reference_kind = 'instantiate' - AND s.name <> r.symbol_name COLLATE BINARY THEN 0 + AND instantiation_fact.name <> + r.symbol_name COLLATE BINARY THEN 0 WHEN r.reference_kind = 'instantiate' - AND s.kind = 'function' - AND s.container_name = s.name COLLATE BINARY + AND instantiation_fact.candidate_kind = 'function' AND ( - {BuildCSharpConstructorParameterCountSql("s")} IS NOT NULL - OR {BuildCSharpConstructorBindingSensitiveSql("s")} = 1 + instantiation_fact.constructor_parameter_count IS NOT NULL + OR instantiation_fact.constructor_binding_sensitive = 1 ) AND ( {CSharpReferenceArgumentCountSql} IS NULL - OR {BuildCSharpConstructorParameterCountSql("s")} IS NULL - OR {BuildCSharpConstructorFamilyBindingSensitiveSql("s")} - OR {BuildCSharpConstructorParameterCountSql("s")} - = {CSharpReferenceArgumentCountSql} + OR instantiation_fact.constructor_parameter_count IS NULL + OR instantiation_fact.lower_rank_family_binding_sensitive = 1 + OR instantiation_fact.constructor_parameter_count = + {CSharpReferenceArgumentCountSql} ) AND ( {CSharpReferenceTypeAritySql} IS NULL - OR {BuildCSharpConstructorTypeAritySql("s")} - = {CSharpReferenceTypeAritySql} + OR instantiation_fact.type_arity = + {CSharpReferenceTypeAritySql} ) THEN 1 WHEN r.reference_kind = 'instantiate' - AND s.kind IN ('class', 'struct', 'record', 'enum', 'delegate') + AND instantiation_fact.candidate_kind IN ( + 'class', + 'struct', + 'record', + 'enum', + 'delegate') AND ( {CSharpReferenceTypeAritySql} IS NULL - OR {BuildCSharpDefinitionTypeAritySql("s")} - = {CSharpReferenceTypeAritySql} - ) - AND s.id = ( - SELECT representative.id - FROM symbols AS representative - JOIN files AS representative_file - ON representative_file.id = representative.file_id - AND representative_file.lang = 'csharp' - WHERE representative.name_folded = s.name_folded - AND representative.name = s.name COLLATE BINARY - AND representative.kind IN ( - 'class', - 'struct', - 'record', - 'enum', - 'delegate') - AND {BuildCSharpTypeIdentitySql("representative")} - = {BuildCSharpTypeIdentitySql("s")} COLLATE BINARY - ORDER BY representative_file.path, - COALESCE(representative.start_line, representative.line), - representative.id - LIMIT 1 + OR instantiation_fact.type_arity = + {CSharpReferenceTypeAritySql} ) + AND instantiation_fact.is_representative = 1 AND ( ( ( - {BuildCSharpConstructorParameterCountSql("s")} IS NOT NULL - OR {BuildCSharpConstructorBindingSensitiveSql("s")} = 1 + instantiation_fact.constructor_parameter_count IS NOT NULL + OR instantiation_fact.constructor_binding_sensitive = 1 ) AND ( {CSharpReferenceArgumentCountSql} IS NULL - OR {BuildCSharpConstructorParameterCountSql("s")} IS NULL - OR {BuildCSharpConstructorFamilyBindingSensitiveSql("s")} - OR {BuildCSharpConstructorParameterCountSql("s")} - = {CSharpReferenceArgumentCountSql} + OR instantiation_fact.constructor_parameter_count IS NULL + OR instantiation_fact.lower_rank_family_binding_sensitive = 1 + OR instantiation_fact.constructor_parameter_count = + {CSharpReferenceArgumentCountSql} ) ) - OR s.kind = 'delegate' + OR instantiation_fact.candidate_kind = 'delegate' OR ( - s.kind = 'enum' + instantiation_fact.candidate_kind = 'enum' AND ( {CSharpReferenceArgumentCountSql} IS NULL OR {CSharpReferenceArgumentCountSql} = 0 ) ) OR ( - s.kind IN ('class', 'record') - AND {BuildCSharpIsValueTypeSql("s")} = 0 - AND {BuildCSharpConstructorParameterCountSql("s")} IS NULL - AND {BuildCSharpConstructorBindingSensitiveSql("s")} = 0 + instantiation_fact.candidate_kind IN ('class', 'record') + AND instantiation_fact.is_value_type = 0 + AND instantiation_fact.constructor_parameter_count IS NULL + AND instantiation_fact.constructor_binding_sensitive = 0 AND ( {CSharpReferenceArgumentCountSql} IS NULL OR {CSharpReferenceArgumentCountSql} = 0 ) - AND NOT EXISTS ( - SELECT 1 - FROM symbols AS explicit_constructor - JOIN files AS constructor_file - ON constructor_file.id = explicit_constructor.file_id - AND constructor_file.lang = 'csharp' - WHERE explicit_constructor.name_folded = s.name_folded - AND explicit_constructor.name = s.name COLLATE BINARY - AND explicit_constructor.kind = 'function' - AND explicit_constructor.container_name = - explicit_constructor.name COLLATE BINARY - AND ( - {BuildCSharpConstructorParameterCountSql("explicit_constructor")} - IS NOT NULL - OR {BuildCSharpConstructorBindingSensitiveSql("explicit_constructor")} = 1 - ) - AND {BuildCSharpConstructorIdentitySql("explicit_constructor")} - = {BuildCSharpTypeIdentitySql("s")} COLLATE BINARY - ) + AND instantiation_fact.has_explicit_constructor = 0 ) OR ( - {BuildCSharpIsValueTypeSql("s")} = 1 + instantiation_fact.is_value_type = 1 AND ( {CSharpReferenceArgumentCountSql} IS NULL OR {CSharpReferenceArgumentCountSql} = 0 ) - AND NOT EXISTS ( - SELECT 1 - FROM symbols AS explicit_zero_constructor - JOIN files AS zero_constructor_file - ON zero_constructor_file.id = explicit_zero_constructor.file_id - AND zero_constructor_file.lang = 'csharp' - WHERE explicit_zero_constructor.name_folded = s.name_folded - AND explicit_zero_constructor.name = s.name COLLATE BINARY - AND explicit_zero_constructor.kind = 'function' - AND explicit_zero_constructor.container_name = - explicit_zero_constructor.name COLLATE BINARY - AND {BuildCSharpConstructorParameterCountSql("explicit_zero_constructor")} - = 0 - AND {BuildCSharpConstructorIdentitySql("explicit_zero_constructor")} - = {BuildCSharpTypeIdentitySql("s")} COLLATE BINARY - ) + AND instantiation_fact.has_explicit_zero_constructor = 0 ) ) THEN 1 WHEN r.reference_kind = 'instantiate' THEN 0 @@ -1263,6 +1401,10 @@ ON s.name_folded IN ( THEN r.symbol_name_folded || 'attribute' END ) JOIN files AS target_file ON target_file.id = s.file_id + LEFT JOIN temp.csharp_instantiation_family_facts AS instantiation_fact + ON source_file.lang = 'csharp' + AND r.reference_kind = 'instantiate' + AND instantiation_fact.symbol_id = s.id WHERE ( (source_file.lang = target_file.lang AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) @@ -1331,6 +1473,10 @@ ON s.name_folded IN ( THEN r.symbol_name_folded || 'attribute' END ) JOIN files AS target_file ON target_file.id = s.file_id + LEFT JOIN temp.csharp_instantiation_family_facts AS instantiation_fact + ON source_file.lang = 'csharp' + AND r.reference_kind = 'instantiate' + AND instantiation_fact.symbol_id = s.id WHERE source_file.lang = 'csharp' AND target_file.lang = 'csharp' AND {CSharpTypeReferenceCandidatePredicateSql} @@ -1390,6 +1536,10 @@ ON s.name_folded IN ( THEN r.symbol_name_folded || 'attribute' END ) JOIN files AS target_file ON target_file.id = s.file_id + LEFT JOIN temp.csharp_instantiation_family_facts AS instantiation_fact + ON source_file.lang = 'csharp' + AND r.reference_kind = 'instantiate' + AND instantiation_fact.symbol_id = s.id LEFT JOIN symbols AS source ON source.id = r.source_symbol_id WHERE ( (source_file.lang = target_file.lang @@ -1630,211 +1780,20 @@ FROM temp.{ReferenceLowerRankCandidateMatchesTable} AS lower_rank_match ); INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) - WITH csharp_instantiation_type_members( - symbol_id, - name_folded, - name, - type_arity, - type_identity, - candidate_kind, - constructor_parameter_count, - constructor_binding_sensitive, - is_value_type, - representative_rank) AS MATERIALIZED ( - SELECT s.id, - s.name_folded, - s.name COLLATE BINARY, - symbol_fact.definition_type_arity, - type_identity_fact.type_identity COLLATE BINARY, - s.kind, - symbol_fact.constructor_parameter_count, - {BuildCSharpConstructorBindingSensitiveSql("s")}, - symbol_fact.is_value_type, - ROW_NUMBER() OVER ( - PARTITION BY s.name_folded, - s.name COLLATE BINARY, - symbol_fact.definition_type_arity, - type_identity_fact.type_identity COLLATE BINARY - ORDER BY target_file.path COLLATE BINARY, - COALESCE(s.start_line, s.line), - s.id) - FROM symbols AS s - JOIN files AS target_file ON target_file.id = s.file_id - JOIN temp.csharp_symbol_facts AS symbol_fact - ON symbol_fact.symbol_id = s.id - JOIN temp.csharp_type_identity_facts AS type_identity_fact - ON type_identity_fact.symbol_id = s.id - WHERE target_file.lang = 'csharp' - AND s.name_folded IS NOT NULL - AND s.kind IN ('class', 'struct', 'record', 'enum', 'delegate') - ), - csharp_unique_instantiation_families( - name_folded, - name, - type_arity, - type_identity) AS MATERIALIZED ( - SELECT type_member.name_folded, - type_member.name COLLATE BINARY, - type_member.type_arity, - MIN(type_member.type_identity COLLATE BINARY) - FROM csharp_instantiation_type_members AS type_member - GROUP BY type_member.name_folded, - type_member.name COLLATE BINARY, - type_member.type_arity - HAVING COUNT(type_member.type_identity) > 0 - AND MIN(type_member.type_identity COLLATE BINARY) - IS MAX(type_member.type_identity COLLATE BINARY) - ), - csharp_instantiation_constructor_members( - symbol_id, - name_folded, - name, - type_arity, - type_identity, - constructor_parameter_count, - constructor_binding_sensitive, - constructor_family_binding_sensitive) AS MATERIALIZED ( - SELECT constructor.id, - unique_family.name_folded, - unique_family.name COLLATE BINARY, - unique_family.type_arity, - unique_family.type_identity COLLATE BINARY, - constructor_fact.constructor_parameter_count, - {BuildCSharpConstructorBindingSensitiveSql("constructor")}, - MAX( - MAX({BuildCSharpConstructorBindingSensitiveSql("constructor")}) OVER ( - PARTITION BY unique_family.name_folded, - unique_family.name COLLATE BINARY, - unique_family.type_arity, - unique_family.type_identity COLLATE BINARY), - primary_member.constructor_binding_sensitive) - FROM csharp_unique_instantiation_families AS unique_family - JOIN csharp_instantiation_type_members AS primary_member - ON primary_member.name_folded = unique_family.name_folded - AND primary_member.name = unique_family.name COLLATE BINARY - AND primary_member.type_arity IS unique_family.type_arity - AND primary_member.type_identity = - unique_family.type_identity COLLATE BINARY - AND primary_member.representative_rank = 1 - CROSS JOIN symbols AS constructor INDEXED BY idx_symbols_name_folded - ON constructor.name_folded = unique_family.name_folded - AND constructor.name = unique_family.name COLLATE BINARY - JOIN files AS constructor_file - ON constructor_file.id = constructor.file_id - AND constructor_file.lang = 'csharp' - JOIN temp.csharp_symbol_facts AS constructor_fact - ON constructor_fact.symbol_id = constructor.id - AND ( - constructor_fact.constructor_parameter_count IS NOT NULL - OR {BuildCSharpConstructorBindingSensitiveSql("constructor")} = 1 - ) - JOIN temp.csharp_constructor_identity_facts AS constructor_identity - ON constructor_identity.symbol_id = constructor.id - AND constructor_identity.type_identity = - unique_family.type_identity COLLATE BINARY - AND constructor_identity.type_arity IS unique_family.type_arity - WHERE constructor.kind = 'function' - AND constructor.container_name = constructor.name COLLATE BINARY - ), - csharp_instantiation_constructor_summary( - name_folded, - name, - type_arity, - type_identity, - has_explicit_constructor, - has_explicit_zero_constructor, - has_binding_sensitive_constructor) AS MATERIALIZED ( - SELECT unique_family.name_folded, - unique_family.name COLLATE BINARY, - unique_family.type_arity, - unique_family.type_identity COLLATE BINARY, - MAX(constructor_member.symbol_id IS NOT NULL), - MAX(COALESCE( - constructor_member.constructor_parameter_count = 0, - 0)), - MAX(COALESCE( - constructor_member.constructor_family_binding_sensitive, - 0)) - FROM csharp_unique_instantiation_families AS unique_family - LEFT JOIN csharp_instantiation_constructor_members AS constructor_member - ON constructor_member.name_folded = unique_family.name_folded - AND constructor_member.name = unique_family.name COLLATE BINARY - AND constructor_member.type_arity IS unique_family.type_arity - AND constructor_member.type_identity = - unique_family.type_identity COLLATE BINARY - GROUP BY unique_family.name_folded, - unique_family.name COLLATE BINARY, - unique_family.type_arity, - unique_family.type_identity COLLATE BINARY - ), - csharp_instantiation_targets( - symbol_id, - name_folded, - name, - type_arity, - type_identity, - candidate_kind, - constructor_parameter_count, - constructor_binding_sensitive, - constructor_family_binding_sensitive, - is_value_type, - has_explicit_constructor, - has_explicit_zero_constructor) AS ( - SELECT constructor_member.symbol_id, - constructor_member.name_folded, - constructor_member.name COLLATE BINARY, - constructor_member.type_arity, - constructor_member.type_identity COLLATE BINARY, - 'function', - constructor_member.constructor_parameter_count, - constructor_member.constructor_binding_sensitive, - constructor_member.constructor_family_binding_sensitive, - 0, - 1, - constructor_member.constructor_parameter_count = 0 - FROM csharp_instantiation_constructor_members AS constructor_member - - UNION ALL - - SELECT type_member.symbol_id, - unique_family.name_folded, - unique_family.name COLLATE BINARY, - unique_family.type_arity, - unique_family.type_identity COLLATE BINARY, - type_member.candidate_kind, - type_member.constructor_parameter_count, - type_member.constructor_binding_sensitive, - MAX( - type_member.constructor_binding_sensitive, - COALESCE( - constructor_summary.has_binding_sensitive_constructor, - 0)), - type_member.is_value_type, - constructor_summary.has_explicit_constructor, - constructor_summary.has_explicit_zero_constructor - FROM csharp_unique_instantiation_families AS unique_family - JOIN csharp_instantiation_type_members AS type_member - ON type_member.name_folded = unique_family.name_folded - AND type_member.name = unique_family.name COLLATE BINARY - AND type_member.type_arity IS unique_family.type_arity - AND type_member.type_identity = - unique_family.type_identity COLLATE BINARY - AND type_member.representative_rank = 1 - JOIN csharp_instantiation_constructor_summary AS constructor_summary - ON constructor_summary.name_folded = unique_family.name_folded - AND constructor_summary.name = unique_family.name COLLATE BINARY - AND constructor_summary.type_arity IS unique_family.type_arity - AND constructor_summary.type_identity = - unique_family.type_identity COLLATE BINARY - ) SELECT r.id, unique_target.symbol_id, 5 FROM symbol_references AS r - JOIN files AS source_file ON source_file.id = r.file_id - JOIN csharp_instantiation_targets AS unique_target + CROSS JOIN files AS source_file ON source_file.id = r.file_id + CROSS JOIN temp.csharp_instantiation_family_facts AS unique_target + INDEXED BY idx_csharp_instantiation_family_facts_lookup ON unique_target.name_folded = r.symbol_name_folded AND unique_target.name = r.symbol_name COLLATE BINARY + AND unique_target.fallback_family_is_unique = 1 + AND ( + unique_target.candidate_kind = 'function' + OR unique_target.is_representative = 1 + ) LEFT JOIN temp.csharp_reference_facts AS reference_fact ON reference_fact.reference_id = r.id WHERE source_file.lang = 'csharp' @@ -1850,7 +1809,7 @@ reference_fact.type_arity IS NULL AND ( reference_fact.argument_count IS NULL OR unique_target.constructor_parameter_count IS NULL - OR unique_target.constructor_family_binding_sensitive = 1 + OR unique_target.fallback_family_binding_sensitive = 1 OR unique_target.constructor_parameter_count = reference_fact.argument_count ) @@ -1864,7 +1823,7 @@ unique_target.constructor_parameter_count IS NOT NULL AND ( reference_fact.argument_count IS NULL OR unique_target.constructor_parameter_count IS NULL - OR unique_target.constructor_family_binding_sensitive = 1 + OR unique_target.fallback_family_binding_sensitive = 1 OR unique_target.constructor_parameter_count = reference_fact.argument_count ) @@ -2071,6 +2030,7 @@ internal static void RebuildRetainedReferenceGraph( RefreshCSharpSymbolFactsFullSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpInstantiationFamilyFactsSql + "\n" + RefreshCSharpPropertyTargetFactsFullSql + "\n" + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceUniqueFamiliesSql + "\n" + @@ -3058,6 +3018,7 @@ internal void RefreshMutualRecursionFlags( RefreshCSharpSymbolFactsFullSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpInstantiationFamilyFactsSql + "\n" + RefreshCSharpPropertyTargetFactsFullSql + "\n" + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceUniqueFamiliesSql + "\n" + @@ -3074,6 +3035,7 @@ internal void RefreshMutualRecursionFlags( RefreshCSharpSymbolFactsScopedSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpInstantiationFamilyFactsSql + "\n" + RefreshCSharpPropertyTargetFactsScopedSql + "\n" + NormalizeCSharpPropertyReceiverReferencesScopedSql + "\n" + RefreshScopedReferenceUniqueFamiliesSql + "\n" + diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index bfe30ab2e0..5d91150ccc 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -260,6 +260,7 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() "csharp_reference_is_member_receiver(", "csharp_definition_type_arity(", "csharp_constructor_parameter_count(", + "csharp_constructor_has_binding_sensitive_parameters(", "csharp_definition_is_value_type(", ]; @@ -292,6 +293,12 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() var constructorIdentityInsert = sql.IndexOf( "INSERT INTO temp.csharp_constructor_identity_facts", StringComparison.Ordinal); + var instantiationFamilyDelete = sql.IndexOf( + "DELETE FROM temp.csharp_instantiation_family_facts", + StringComparison.Ordinal); + var instantiationFamilyInsert = sql.IndexOf( + "INSERT INTO temp.csharp_instantiation_family_facts", + StringComparison.Ordinal); var propertyTargetDelete = sql.IndexOf( "DELETE FROM temp.csharp_property_target_facts", StringComparison.Ordinal); @@ -314,7 +321,9 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() && typeIdentityDelete < typeIdentityInsert && typeIdentityInsert < constructorIdentityDelete && constructorIdentityDelete < constructorIdentityInsert - && constructorIdentityInsert < propertyTargetDelete + && constructorIdentityInsert < instantiationFamilyDelete + && instantiationFamilyDelete < instantiationFamilyInsert + && instantiationFamilyInsert < propertyTargetDelete && propertyTargetDelete < propertyTargetInsert && propertyTargetInsert < normalization && normalization < candidates, @@ -356,6 +365,66 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() } _writer.RefreshMutualRecursionFlags(); + var instantiationFactStages = DbWriter.CSharpInstantiationFamilyFactSqlForTesting; + Assert.Equal( + ["full", "scoped", "retained"], + instantiationFactStages.Select(static stage => stage.Scope)); + foreach (var (_, sql) in instantiationFactStages) + { + Assert.Contains("csharp_instantiation_type_members(", sql, StringComparison.Ordinal); + Assert.Contains("csharp_instantiation_constructor_members(", sql, StringComparison.Ordinal); + Assert.Contains("csharp_instantiation_members(", sql, StringComparison.Ordinal); + Assert.DoesNotContain("CORRELATED", sql, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("symbols AS binding_sensitive_constructor", sql, StringComparison.Ordinal); + Assert.DoesNotContain( + "csharp_constructor_has_binding_sensitive_parameters(", + sql, + StringComparison.Ordinal); + + var insert = Assert.Single( + sql.Split( + ';', + StringSplitOptions.RemoveEmptyEntries + | StringSplitOptions.TrimEntries) + .Where(static statement => statement.Contains( + "INSERT INTO temp.csharp_instantiation_family_facts", + StringComparison.Ordinal))); + var plan = ReadQueryPlanDetails(_db.Connection, insert); + Assert.True( + plan.Any(static detail => detail.Contains( + "SEARCH symbol USING INTEGER PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)), + string.Join(Environment.NewLine, plan)); + Assert.True( + plan.Any(static detail => detail.Contains( + "SEARCH constructor USING INTEGER PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)), + string.Join(Environment.NewLine, plan)); + Assert.Contains( + plan, + static detail => detail.Contains( + "SEARCH symbol_fact USING PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + plan, + static detail => detail.Contains( + "SEARCH constructor_fact USING PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + plan, + static detail => detail.Contains( + "idx_symbols_kind", + StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + plan, + static detail => detail.Contains("AUTOMATIC", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + plan, + static detail => detail.Contains( + "CORRELATED SCALAR SUBQUERY", + StringComparison.OrdinalIgnoreCase)); + } + using var candidateScope = _writer.BeginReferenceGraphRefreshScope(); var candidateStages = DbWriter.CSharpGraphCandidateSqlForTesting; Assert.Equal(["full", "scoped", "retained"], candidateStages.Select(static stage => stage.Scope)); @@ -373,7 +442,8 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() StringComparison.Ordinal); Assert.Contains("LEFT JOIN symbols AS source", sql, StringComparison.Ordinal); Assert.Contains("temp.csharp_type_identity_facts", sql, StringComparison.Ordinal); - Assert.Contains("temp.csharp_constructor_identity_facts", sql, StringComparison.Ordinal); + Assert.Contains("temp.csharp_instantiation_family_facts", sql, StringComparison.Ordinal); + Assert.DoesNotContain("temp.csharp_constructor_identity_facts", sql, StringComparison.Ordinal); Assert.Contains( "csharp_type_reference_members(", sql, @@ -417,32 +487,28 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) .Where(static statement => statement.Contains( - "csharp_instantiation_type_members(", + "unique_target.fallback_family_is_unique = 1", StringComparison.Ordinal))); - Assert.Contains( + Assert.DoesNotContain( "csharp_unique_instantiation_families(", instantiationStatement, StringComparison.Ordinal); - Assert.Contains( + Assert.DoesNotContain( "csharp_instantiation_constructor_members(", instantiationStatement, StringComparison.Ordinal); - Assert.Contains( + Assert.DoesNotContain( "csharp_instantiation_constructor_summary(", instantiationStatement, StringComparison.Ordinal); - Assert.Contains( + Assert.DoesNotContain( "csharp_instantiation_targets(", instantiationStatement, StringComparison.Ordinal); - Assert.Equal(4, CountOccurrences(instantiationStatement, "AS MATERIALIZED")); - Assert.Equal( - 1, - CountOccurrences( - instantiationStatement, - "JOIN symbols AS constructor")); + Assert.Equal(0, CountOccurrences(instantiationStatement, "AS MATERIALIZED")); + Assert.DoesNotContain("JOIN symbols AS constructor", instantiationStatement, StringComparison.Ordinal); Assert.Contains( - "constructor_identity.type_arity IS unique_family.type_arity", + "idx_csharp_instantiation_family_facts_lookup", instantiationStatement, StringComparison.Ordinal); Assert.DoesNotContain( @@ -453,28 +519,75 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() "symbols AS explicit_zero_constructor", instantiationStatement, StringComparison.Ordinal); + Assert.DoesNotContain( + "symbols AS binding_sensitive_constructor", + instantiationStatement, + StringComparison.Ordinal); + Assert.DoesNotContain( + "symbols AS representative", + instantiationStatement, + StringComparison.Ordinal); var instantiationPlan = ReadQueryPlanDetails( _db.Connection, instantiationStatement); - Assert.Equal( - 1, - instantiationPlan.Count(static detail => detail.Contains( - "CORRELATED SCALAR SUBQUERY", - StringComparison.OrdinalIgnoreCase))); Assert.Contains( instantiationPlan, static detail => detail.Contains( - "SEARCH constructor USING INDEX idx_symbols_name_folded", + "SEARCH unique_target USING INDEX idx_csharp_instantiation_family_facts_lookup", StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + instantiationPlan, + static detail => detail.Contains( + "name_folded=? AND name=? AND fallback_family_is_unique=?", + StringComparison.OrdinalIgnoreCase)); + if (scope == "scoped") + { + Assert.Contains( + instantiationPlan, + static detail => detail.StartsWith( + "SCAN dirty_reference", + StringComparison.OrdinalIgnoreCase)); + Assert.Contains( + instantiationPlan, + static detail => detail.Contains( + "SEARCH r USING INTEGER PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)); + } + else + { + Assert.True( + instantiationPlan.Count(static detail => detail.StartsWith( + "SCAN r", + StringComparison.OrdinalIgnoreCase)) == 1, + string.Join(Environment.NewLine, instantiationPlan)); + } Assert.DoesNotContain( instantiationPlan, - static detail => detail.Equals( - "SCAN constructor", - StringComparison.OrdinalIgnoreCase) - || detail.StartsWith( - "SCAN constructor ", + static detail => detail.Contains("AUTOMATIC", StringComparison.OrdinalIgnoreCase)); + + var lowerRankStatements = sql.Split( + ';', + StringSplitOptions.RemoveEmptyEntries + | StringSplitOptions.TrimEntries) + .Where(static statement => statement.Contains( + "LEFT JOIN temp.csharp_instantiation_family_facts AS instantiation_fact", + StringComparison.Ordinal)) + .ToArray(); + Assert.Equal(3, lowerRankStatements.Length); + foreach (var lowerRankStatement in lowerRankStatements) + { + var lowerRankPlan = ReadQueryPlanDetails(_db.Connection, lowerRankStatement); + Assert.Contains( + lowerRankPlan, + static detail => detail.Contains( + "SEARCH instantiation_fact USING PRIMARY KEY", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain( + lowerRankPlan, + static detail => detail.Contains("AUTOMATIC", StringComparison.OrdinalIgnoreCase) + && detail.Contains("instantiation_fact", StringComparison.OrdinalIgnoreCase)); + } } static int CountOccurrences(string text, string value) @@ -718,6 +831,36 @@ public void CSharpInstantiationFallback_SetBasedFamiliesPreserveSemanticBoundari "delegate", "Factory", "public delegate void Factory(int value)"); + AddTypeFile( + "types/OptionalFamily.cs", + "class", + "OptionalFamily", + "public class OptionalFamily", + constructors: + [ + (4, "public OptionalFamily(int value)"), + (5, "public OptionalFamily(string value = \"\")"), + ]); + AddTypeFile( + "types/ParamsFamily.cs", + "class", + "ParamsFamily", + "public class ParamsFamily", + constructors: + [ + (4, "public ParamsFamily(int value)"), + (5, "public ParamsFamily(params string[] values)"), + ]); + AddTypeFile( + "types/OptionalAttributeFamily.cs", + "class", + "OptionalAttributeFamily", + "public class OptionalAttributeFamily", + constructors: + [ + (4, "public OptionalAttributeFamily(int value)"), + (5, "public OptionalAttributeFamily([System.Runtime.InteropServices.Optional] string value)"), + ]); AddType( callerFileId, "class", @@ -746,6 +889,9 @@ public void CSharpInstantiationFallback_SetBasedFamiliesPreserveSemanticBoundari Instantiation("Choice", 16, "new Choice();"), Instantiation("Choice", 17, "new Choice(1);"), Instantiation("Factory", 18, "new Factory(Handler);"), + Instantiation("OptionalFamily", 19, "new OptionalFamily(1, 2, 3);"), + Instantiation("ParamsFamily", 20, "new ParamsFamily(1, 2, 3);"), + Instantiation("OptionalAttributeFamily", 21, "new OptionalAttributeFamily(1, 2, 3);"), ], refreshMutualRecursionFlags: false); @@ -776,6 +922,32 @@ public void CSharpInstantiationFallback_SetBasedFamiliesPreserveSemanticBoundari Assert.Equal("types/Choice.cs:enum:1", ReadCandidates(line: 16)); Assert.Empty(ReadCandidates(line: 17)); Assert.Equal("types/Factory.cs:delegate:1", ReadCandidates(line: 18)); + Assert.Equal( + "types/OptionalFamily.cs:function:4|types/OptionalFamily.cs:function:5", + ReadCandidates(line: 19)); + Assert.Equal( + "types/ParamsFamily.cs:function:4|types/ParamsFamily.cs:function:5", + ReadCandidates(line: 20)); + Assert.Equal( + "types/OptionalAttributeFamily.cs:function:4|types/OptionalAttributeFamily.cs:function:5", + ReadCandidates(line: 21)); + Assert.Equal( + 2, + ExecuteScalarLong(""" + SELECT COUNT(*) + FROM temp.csharp_instantiation_family_facts AS fact + JOIN symbols AS symbol ON symbol.id = fact.symbol_id + WHERE symbol.name = 'Partial' + AND fact.fallback_family_is_unique = 1 + """)); + Assert.Equal( + "2|0", + ExecuteScalarString(""" + SELECT COUNT(*) || '|' || SUM(fact.fallback_family_is_unique) + FROM temp.csharp_instantiation_family_facts AS fact + JOIN symbols AS symbol ON symbol.id = fact.symbol_id + WHERE symbol.name = 'Ambiguous' + """)); long AddTypeFile( string path, @@ -868,6 +1040,119 @@ JOIN symbol_references AS reference } } + [Fact] + public void CSharpInstantiationFamilies_KeepLowerRankAndFallbackPartialSensitivitySeparate() + { + var representativeFileId = UpsertTestFile( + "types/PartialBinding.A.cs", + "partial-binding-a"); + var nonRepresentativeFileId = UpsertTestFile( + "types/PartialBinding.B.cs", + "partial-binding-b"); + var qualifiedCallerFileId = UpsertTestFile( + "calls/QualifiedPartialBinding.cs", + "qualified-partial-binding"); + var fallbackCallerFileId = UpsertTestFile( + "calls/FallbackPartialBinding.cs", + "fallback-partial-binding"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = representativeFileId, + Kind = "class", + Name = "PartialBinding", + Line = 1, + StartLine = 1, + EndLine = 20, + Signature = "public partial class PartialBinding", + ContainerQualifiedName = "Demo", + }, + new SymbolRecord + { + FileId = representativeFileId, + Kind = "function", + Name = "PartialBinding", + Line = 4, + StartLine = 4, + EndLine = 4, + Signature = "public PartialBinding(int value)", + ContainerKind = "class", + ContainerName = "PartialBinding", + ContainerQualifiedName = "Demo.PartialBinding", + }, + new SymbolRecord + { + FileId = nonRepresentativeFileId, + Kind = "class", + Name = "PartialBinding", + Line = 1, + StartLine = 1, + EndLine = 20, + Signature = "public partial class PartialBinding(int value = 0)", + ContainerQualifiedName = "Demo", + }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = qualifiedCallerFileId, + SymbolName = "PartialBinding", + ReferenceKind = "instantiate", + Line = 1, + Column = 5, + Context = "new PartialBinding(1, 2);", + TargetQualifier = "Demo", + }, + new ReferenceRecord + { + FileId = fallbackCallerFileId, + SymbolName = "PartialBinding", + ReferenceKind = "instantiate", + Line = 1, + Column = 5, + Context = "new PartialBinding(1, 2);", + }, + ], refreshMutualRecursionFlags: false); + + _writer.RefreshMutualRecursionFlags(); + + Assert.Equal( + "function|1|0|0", + ExecuteScalarString(""" + SELECT fact.candidate_kind || '|' || + fact.lower_rank_family_binding_sensitive || '|' || + fact.fallback_family_binding_sensitive || '|' || + fact.is_representative + FROM temp.csharp_instantiation_family_facts AS fact + JOIN symbols AS symbol ON symbol.id = fact.symbol_id + WHERE symbol.name = 'PartialBinding' + AND symbol.kind = 'function' + """)); + Assert.Equal( + 1, + ExecuteScalarLong(""" + SELECT COUNT(*) + FROM symbol_reference_candidates AS candidate + JOIN symbol_references AS reference + ON reference.id = candidate.reference_id + JOIN symbols AS target ON target.id = candidate.symbol_id + JOIN files AS source_file ON source_file.id = reference.file_id + WHERE source_file.path = 'calls/QualifiedPartialBinding.cs' + AND target.kind = 'function' + AND candidate.scope_rank = 0 + """)); + Assert.Equal( + 0, + ExecuteScalarLong(""" + SELECT COUNT(*) + FROM symbol_reference_candidates AS candidate + JOIN symbol_references AS reference + ON reference.id = candidate.reference_id + JOIN files AS source_file ON source_file.id = reference.file_id + WHERE source_file.path = 'calls/FallbackPartialBinding.cs' + """)); + } + [Fact] public void CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember() { @@ -1211,6 +1496,15 @@ public void CSharpGraphIdentityFacts_PreserveFullScopedAndRetainedConstructorRes }, CreateInstantiation(localAFileId, "Hidden", "new Hidden();", line: 15), CreateInstantiation(localBFileId, "Hidden", "new Hidden();", line: 15), + CreateInstantiation(orphanFileId, "Orphan", "new Orphan();", line: 2), + new ReferenceRecord + { + FileId = orphanFileId, + SymbolName = "Orphan", + ReferenceKind = "instantiate", + Line = 3, + Column = 5, + }, CreateInstantiation( callerFileId, "PrimaryBox", @@ -1253,6 +1547,50 @@ FROM symbol_references AS reference AND target.kind = 'function' AND source_file.path = target_file.path """)); + Assert.Equal( + "function|0|0|0|1|0", + ExecuteScalarString(""" + SELECT fact.candidate_kind || '|' || + fact.lower_rank_family_binding_sensitive || '|' || + COALESCE(CAST(fact.constructor_parameter_count AS TEXT), 'null') || '|' || + fact.constructor_binding_sensitive || '|' || + fact.has_explicit_constructor || '|' || + fact.fallback_family_is_unique + FROM temp.csharp_instantiation_family_facts AS fact + JOIN symbols AS symbol ON symbol.id = fact.symbol_id + WHERE symbol.name = 'Orphan' + """)); + Assert.Equal( + "function|3", + ExecuteScalarString(""" + SELECT target.kind || '|' || candidate.scope_rank + FROM symbol_reference_candidates AS candidate + JOIN symbol_references AS reference + ON reference.id = candidate.reference_id + JOIN symbols AS target ON target.id = candidate.symbol_id + WHERE reference.symbol_name = 'Orphan' + AND reference.line = 3 + AND target.kind = 'function' + """)); + Assert.Equal( + 0, + ExecuteScalarLong(""" + SELECT COUNT(*) + FROM symbol_reference_candidates AS candidate + JOIN symbol_references AS reference + ON reference.id = candidate.reference_id + WHERE reference.symbol_name = 'Orphan' + AND reference.line = 2 + """)); + Assert.Equal( + 3, + ExecuteScalarLong(""" + SELECT COUNT(*) + FROM temp.csharp_instantiation_family_facts AS fact + JOIN symbols AS symbol ON symbol.id = fact.symbol_id + WHERE symbol.name = 'Gadget' + AND fact.fallback_family_is_unique = 1 + """)); Assert.Equal( "proj/primary-generic.cs|class", ExecuteScalarString(""" @@ -1349,6 +1687,23 @@ FROM temp.csharp_type_identity_facts AS type_fact JOIN symbols AS symbol ON symbol.id = type_fact.symbol_id WHERE symbol.name = 'Hidden' """)); + Assert.Equal( + 0, + ExecuteScalarLong(""" + SELECT COUNT(*) + FROM temp.csharp_instantiation_family_facts AS fact + JOIN symbols AS symbol ON symbol.id = fact.symbol_id + WHERE symbol.name = 'Hidden' + """)); + Assert.Equal( + 3, + ExecuteScalarLong(""" + SELECT COUNT(*) + FROM temp.csharp_instantiation_family_facts AS fact + JOIN symbols AS symbol ON symbol.id = fact.symbol_id + WHERE symbol.name = 'Gadget' + AND fact.fallback_family_is_unique = 1 + """)); _writer.RefreshMutualRecursionFlags(); var fullSnapshot = ReadReferenceGraphSemanticSnapshot(); @@ -1499,6 +1854,20 @@ SELECT COUNT(*) FROM temp.csharp_type_identity_facts WHERE symbol_id = {freshSymbolId.ToString(CultureInfo.InvariantCulture)} """)); + Assert.Equal( + 1, + ExecuteScalarLong($""" + SELECT COUNT(*) + FROM temp.csharp_instantiation_family_facts + WHERE symbol_id = {staleSymbolId.ToString(CultureInfo.InvariantCulture)} + """)); + Assert.Equal( + 0, + ExecuteScalarLong($""" + SELECT COUNT(*) + FROM temp.csharp_instantiation_family_facts + WHERE symbol_id = {freshSymbolId.ToString(CultureInfo.InvariantCulture)} + """)); ExecuteNonQuery(_db.Connection, "DROP TRIGGER fail_csharp_identity_fact_refresh"); _writer.RefreshMutualRecursionFlags(); @@ -1517,6 +1886,20 @@ SELECT COUNT(*) FROM temp.csharp_type_identity_facts WHERE symbol_id = {freshSymbolId.ToString(CultureInfo.InvariantCulture)} """)); + Assert.Equal( + 0, + ExecuteScalarLong($""" + SELECT COUNT(*) + FROM temp.csharp_instantiation_family_facts + WHERE symbol_id = {staleSymbolId.ToString(CultureInfo.InvariantCulture)} + """)); + Assert.Equal( + 1, + ExecuteScalarLong($""" + SELECT COUNT(*) + FROM temp.csharp_instantiation_family_facts + WHERE symbol_id = {freshSymbolId.ToString(CultureInfo.InvariantCulture)} + """)); Assert.Equal("resolved", ReadReferenceResolutionState(stableFileId)); } @@ -1839,16 +2222,16 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() var candidateSql = DbWriter.RefreshScopedReferenceCandidatesSqlForTesting; Assert.DoesNotContain("AND s.name_folded IS NOT NULL", candidateSql, StringComparison.Ordinal); - Assert.Contains( + Assert.DoesNotContain( "FROM temp.reference_graph_lookup_names AS lookup_name", candidateSql, StringComparison.Ordinal); Assert.Contains( - "CROSS JOIN symbols AS s INDEXED BY idx_symbols_name_folded", + "CROSS JOIN temp.csharp_instantiation_family_facts AS unique_target", candidateSql, StringComparison.Ordinal); Assert.Contains( - "AND s.name_folded = lookup_name.name_folded", + "unique_target.fallback_family_is_unique = 1", candidateSql, StringComparison.Ordinal); Assert.Equal( @@ -1913,26 +2296,25 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() Assert.Contains(candidatePlans, static detail => detail.Contains( "SEARCH type_identity_fact USING PRIMARY KEY", StringComparison.OrdinalIgnoreCase)); - Assert.Contains(candidatePlans, static detail => detail.Contains( - "SEARCH constructor_identity_fact USING PRIMARY KEY", - StringComparison.OrdinalIgnoreCase)); Assert.DoesNotContain(candidatePlans, static detail => detail.Equals("SCAN type_identity_fact", StringComparison.OrdinalIgnoreCase) - || detail.StartsWith("SCAN type_identity_fact ", StringComparison.OrdinalIgnoreCase) - || detail.Equals("SCAN constructor_identity_fact", StringComparison.OrdinalIgnoreCase) - || detail.StartsWith("SCAN constructor_identity_fact ", StringComparison.OrdinalIgnoreCase)); + || detail.StartsWith("SCAN type_identity_fact ", StringComparison.OrdinalIgnoreCase)); var instantiateStatement = Assert.Single(candidateInserts.Where(static statement => statement.Contains( - "FROM temp.reference_graph_lookup_names AS lookup_name", + "unique_target.fallback_family_is_unique = 1", StringComparison.Ordinal))); var instantiatePlan = ReadQueryPlanDetails(_db.Connection, instantiateStatement); Assert.Contains(instantiatePlan, static detail => detail.Contains( - "idx_symbols_name_folded", + "SEARCH unique_target USING INDEX idx_csharp_instantiation_family_facts_lookup", StringComparison.OrdinalIgnoreCase)); Assert.Contains(instantiatePlan, static detail => detail.Contains( - "SEARCH lookup_name USING PRIMARY KEY", + "name_folded=? AND name=? AND fallback_family_is_unique=?", + StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(instantiatePlan, static detail => detail.Contains( + "AUTOMATIC", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(instantiateStatement, "JOIN symbols AS constructor", StringComparison.Ordinal); var csharpTypeStatement = Assert.Single(candidateInserts.Where(static statement => statement.Contains( @@ -2802,6 +3184,9 @@ public void RefreshReferenceIdentities_CancellationAfterCandidatesRollsBackAndRe "src/candidate-boundary.py", "python", $"candidate-boundary-{forceFullRefresh}"); + var csharpFileId = UpsertTestFile( + "src/CandidateBoundary.cs", + $"csharp-candidate-boundary-{forceFullRefresh}"); _writer.InsertSymbols([ new SymbolRecord { @@ -2812,6 +3197,17 @@ public void RefreshReferenceIdentities_CancellationAfterCandidatesRollsBackAndRe StartLine = 1, EndLine = 1, }, + new SymbolRecord + { + FileId = csharpFileId, + Kind = "class", + Name = "CandidateBoundaryType", + Line = 1, + StartLine = 1, + EndLine = 10, + Signature = "public partial class CandidateBoundaryType", + ContainerQualifiedName = "Demo", + }, ]); _writer.InsertReferences([ new ReferenceRecord @@ -2827,6 +3223,7 @@ public void RefreshReferenceIdentities_CancellationAfterCandidatesRollsBackAndRe _writer.RefreshMutualRecursionFlags(); Assert.Equal(1, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); Assert.Equal("resolved", ReadReferenceResolutionState(fileId)); + Assert.Equal(1, ReadCSharpFamilyFactCount()); using var scope = _writer.BeginReferenceGraphRefreshScope( forceFullRefresh: forceFullRefresh); @@ -2842,6 +3239,17 @@ public void RefreshReferenceIdentities_CancellationAfterCandidatesRollsBackAndRe StartLine = 2, EndLine = 2, }, + new SymbolRecord + { + FileId = csharpFileId, + Kind = "class", + Name = "CandidateBoundaryType", + Line = 20, + StartLine = 20, + EndLine = 30, + Signature = "public partial class CandidateBoundaryType", + ContainerQualifiedName = "Demo", + }, ]); transaction.Commit(); } @@ -2857,6 +3265,7 @@ public void RefreshReferenceIdentities_CancellationAfterCandidatesRollsBackAndRe Assert.Equal( 2, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + Assert.Equal(2, ReadCSharpFamilyFactCount()); cancellation.Cancel(); previousHook?.Invoke(); }; @@ -2868,6 +3277,7 @@ public void RefreshReferenceIdentities_CancellationAfterCandidatesRollsBackAndRe Assert.Equal(1, boundaryCount); Assert.Equal(1, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); Assert.Equal("resolved", ReadReferenceResolutionState(fileId)); + Assert.Equal(1, ReadCSharpFamilyFactCount()); } finally { @@ -2877,6 +3287,15 @@ public void RefreshReferenceIdentities_CancellationAfterCandidatesRollsBackAndRe _writer.RefreshMutualRecursionFlags(); Assert.Equal(2, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); Assert.Equal("resolved_group", ReadReferenceResolutionState(fileId)); + Assert.Equal(2, ReadCSharpFamilyFactCount()); + + long ReadCSharpFamilyFactCount() + => ExecuteScalarLong(""" + SELECT COUNT(*) + FROM temp.csharp_instantiation_family_facts AS fact + JOIN symbols AS symbol ON symbol.id = fact.symbol_id + WHERE symbol.name = 'CandidateBoundaryType' + """); } [Fact] @@ -2886,6 +3305,9 @@ public void RebuildRetainedReferenceGraph_CancellationAfterCandidatesRollsBackAn "src/retained-candidate-boundary.py", "python", "retained-candidate-boundary"); + var csharpFileId = UpsertTestFile( + "src/RetainedCandidateBoundary.cs", + "csharp-retained-candidate-boundary"); _writer.InsertSymbols([ new SymbolRecord { @@ -2896,6 +3318,17 @@ public void RebuildRetainedReferenceGraph_CancellationAfterCandidatesRollsBackAn StartLine = 1, EndLine = 1, }, + new SymbolRecord + { + FileId = csharpFileId, + Kind = "class", + Name = "RetainedCandidateBoundaryType", + Line = 1, + StartLine = 1, + EndLine = 10, + Signature = "public partial class RetainedCandidateBoundaryType", + ContainerQualifiedName = "Demo", + }, ]); _writer.InsertReferences([ new ReferenceRecord @@ -2910,6 +3343,7 @@ public void RebuildRetainedReferenceGraph_CancellationAfterCandidatesRollsBackAn ], refreshMutualRecursionFlags: false); _writer.RefreshMutualRecursionFlags(); Assert.Equal(1, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + Assert.Equal(1, ReadCSharpFamilyFactCount()); _writer.InsertSymbols([ new SymbolRecord @@ -2921,6 +3355,17 @@ public void RebuildRetainedReferenceGraph_CancellationAfterCandidatesRollsBackAn StartLine = 2, EndLine = 2, }, + new SymbolRecord + { + FileId = csharpFileId, + Kind = "class", + Name = "RetainedCandidateBoundaryType", + Line = 20, + StartLine = 20, + EndLine = 30, + Signature = "public partial class RetainedCandidateBoundaryType", + ContainerQualifiedName = "Demo", + }, ]); var previousHook = DbWriter.ReferenceCandidateRefreshCompletedForTesting; @@ -2932,6 +3377,7 @@ public void RebuildRetainedReferenceGraph_CancellationAfterCandidatesRollsBackAn Assert.Equal( 2, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + Assert.Equal(2, ReadCSharpFamilyFactCount()); cancellation.Cancel(); previousHook?.Invoke(); }; @@ -2949,6 +3395,7 @@ public void RebuildRetainedReferenceGraph_CancellationAfterCandidatesRollsBackAn } Assert.Equal(1, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); + Assert.Equal(1, ReadCSharpFamilyFactCount()); using (var retry = _db.Connection.BeginTransaction()) { DbWriter.RebuildRetainedReferenceGraph( @@ -2959,6 +3406,15 @@ public void RebuildRetainedReferenceGraph_CancellationAfterCandidatesRollsBackAn } Assert.Equal(2, ExecuteScalarLong("SELECT COUNT(*) FROM symbol_reference_candidates")); Assert.Equal("resolved_group", ReadReferenceResolutionState(fileId)); + Assert.Equal(2, ReadCSharpFamilyFactCount()); + + long ReadCSharpFamilyFactCount() + => ExecuteScalarLong(""" + SELECT COUNT(*) + FROM temp.csharp_instantiation_family_facts AS fact + JOIN symbols AS symbol ON symbol.id = fact.symbol_id + WHERE symbol.name = 'RetainedCandidateBoundaryType' + """); } [Fact]