Reduce rebuild memory amplification

DEVELOPER_GUIDE.md

@@ -1240,6 +1240,8 @@ When `--db <path>` is omitted, cdidx resolves the SQLite location from a data di
 
 Every `DbContext` connection sets `PRAGMA cache_size=-65536` (64 MiB), `PRAGMA temp_store=MEMORY`, and on 64-bit processes `PRAGMA mmap_size=268435456` (256 MiB). These are connection-scoped query-performance knobs; they do not alter the on-disk schema and are skipped only where SQLite cannot apply them.
 
+High-churn index runs that enter the bulk-load path temporarily set `mmap_size=0` after the input-snapshot validation barrier. This prevents the SQLite mapping from overlapping the largest managed reference-graph working set. The configured mapping is restored after all write scopes unwind, including failure and cancellation paths; ordinary queries, no-op indexing, and low-churn incremental runs retain the configured value.
+
 Operators can override the defaults with environment variables:
 
 | Variable | Default | Meaning |
@@ -4871,6 +4873,8 @@ apply 時は `PRAGMA optimize` を実行します。
 
 すべての `DbContext` connection は `PRAGMA cache_size=-65536` (64 MiB)、`PRAGMA temp_store=MEMORY`、64-bit process では `PRAGMA mmap_size=268435456` (256 MiB) を設定する。これらは connection-scoped な query-performance knob であり、on-disk schema は変更せず、SQLite が適用できない場合だけ skip される。
 
+bulk-load 経路に入る高 churn な index run は、input-snapshot validation barrier の通過後に一時的に `mmap_size=0` を設定する。これにより SQLite mapping と最大の managed reference-graph working set が重ならないようにする。設定済みの mapping は failure / cancellation を含め、すべての write scope が unwind した後に復元される。通常 query、no-op indexing、低 churn の incremental run は設定値を維持する。
+
 operator は environment variable で既定値を上書きできる。
 
 | Variable | Default | Meaning |

changelog.d/unreleased/5056.changed.md

@@ -0,0 +1,18 @@
+---
+category: changed
+issues:
+  - 5056
+affected:
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs
+  - src/CodeIndex/Database/SqliteMmapBulkWriteGuard.cs
+  - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs
+  - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs
+---
+
+## English
+
+- **Reduced full-index rebuild allocation and peak working-set amplification (#5056)** — repeated C# same-line recovery decisions are now reused, and SQLite memory mapping is temporarily disabled during high-churn bulk loads before the configured value is restored.
+
+## 日本語
+
+- **full index rebuild の allocation と peak working set の増幅を削減しました (#5056)** — C# の同一行 recovery 判定を再利用し、高 churn の bulk load 中だけ SQLite memory mapping を一時停止した後、設定値を復元します。

src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs

@@ -813,6 +813,7 @@ void RecordFullScanTargetStatSkip(int fileIndex, IndexedFileStatReuseResult exis
         // outer write scopes immediately afterwards so no durable readiness, evidence, purge,
         // or file mutation can precede the validation above.
         // scan snapshot の write前 authority barrier 通過直後に outer write scope を開始する。
+        using var mmapBulkWrite = SqliteMmapBulkWriteGuard.Start(writer, useFtsBulkLoad);
         if (options.Rebuild)
             db.RepairIncompleteBatchReadiness();
         using var referenceGraphRefresh = writer.BeginReferenceGraphRefreshScope(

src/CodeIndex/Database/SqliteMmapBulkWriteGuard.cs

@@ -0,0 +1,73 @@
+using System.Globalization;
+using Microsoft.Data.Sqlite;
+
+namespace CodeIndex.Database;
+
+/// <summary>
+/// Temporarily disables SQLite memory-mapped I/O while a high-churn index run holds its
+/// largest managed reference-graph structures. The configured mapping is restored after
+/// the write scopes unwind, including failure and cancellation paths.
+/// </summary>
+internal sealed class SqliteMmapBulkWriteGuard : IDisposable
+{
+    private readonly long _restoreMmapSizeBytes;
+    private SqliteConnection? _connection;
+
+    private SqliteMmapBulkWriteGuard(
+        SqliteConnection connection,
+        long restoreMmapSizeBytes)
+    {
+        _connection = connection;
+        _restoreMmapSizeBytes = restoreMmapSizeBytes;
+    }
+
+    internal static SqliteMmapBulkWriteGuard? Start(
+        DbWriter writer,
+        bool enabled)
+    {
+        if (!enabled || !Environment.Is64BitProcess)
+            return null;
+
+        var connection = writer.Connection;
+        var configuredMmapSizeBytes = ReadMmapSizeBytes(connection);
+        if (configuredMmapSizeBytes <= 0)
+            return null;
+
+        var appliedMmapSizeBytes = SetMmapSizeBytes(connection, 0);
+        return appliedMmapSizeBytes == 0
+            ? new SqliteMmapBulkWriteGuard(connection, configuredMmapSizeBytes)
+            : null;
+    }
+
+    public void Dispose()
+    {
+        var connection = _connection;
+        if (connection == null)
+            return;
+
+        try
+        {
+            SetMmapSizeBytes(connection, _restoreMmapSizeBytes);
+        }
+        finally
+        {
+            _connection = null;
+        }
+    }
+
+    private static long ReadMmapSizeBytes(SqliteConnection connection)
+    {
+        using var command = SqliteConnectionPolicy.CreateCommand(connection);
+        command.CommandText = "PRAGMA mmap_size";
+        return Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture);
+    }
+
+    private static long SetMmapSizeBytes(
+        SqliteConnection connection,
+        long mmapSizeBytes)
+    {
+        using var command = SqliteConnectionPolicy.CreateCommand(connection);
+        command.CommandText = DbPragmaPolicy.MmapSizePragmaSql(mmapSizeBytes);
+        return Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture);
+    }
+}

src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs

@@ -172,6 +172,11 @@ private static List<SymbolRecord> ExtractCore(
                 var stopAfterFirstPatternMatch = false;
                 var restartPatternScanOffset = -1;
                 CSharpPropertyMatchCandidate? csharpPropertyCandidateForLine = null;
+                bool? deferCSharpBracePropertyAtPatternStart = null;
+                bool? deferCSharpFunctionAtPatternStart = null;
+                bool? deferCSharpEventAtPatternStart = null;
+                bool? deferCSharpDelegateAtPatternStart = null;
+                bool? recoverableCSharpPatternAtPatternStart = null;
                 foreach (var pattern in patterns)
                 {
                     if (prologClauseContinuationLines?[i] == true
@@ -267,22 +272,34 @@ private static List<SymbolRecord> ExtractCore(
                             if (lang == "csharp"
                                 && pattern.Kind == "property"
                                 && pattern.BodyStyle == BodyStyle.Brace
-                                && ShouldDeferCSharpBracePropertySameLineAdvance(matchLine, lineOffset))
+                                && (lineOffset != patternStartOffset
+                                    ? ShouldDeferCSharpBracePropertySameLineAdvance(matchLine, lineOffset)
+                                    : deferCSharpBracePropertyAtPatternStart ??=
+                                        ShouldDeferCSharpBracePropertySameLineAdvance(matchLine, lineOffset)))
                             {
                                 break;
                             }
 
                             if (lang == "csharp"
                                 && pattern.Kind == "function"
-                                && ShouldDeferCSharpFunctionSameLineAdvance(matchLine, lineOffset))
+                                && (lineOffset != patternStartOffset
+                                    ? ShouldDeferCSharpFunctionSameLineAdvance(matchLine, lineOffset)
+                                    : deferCSharpFunctionAtPatternStart ??=
+                                        ShouldDeferCSharpFunctionSameLineAdvance(matchLine, lineOffset)))
                             {
                                 break;
                             }
 
                             if (lang == "csharp"
                                 && pattern.Kind is "event" or "delegate"
                                 && pattern.BodyStyle == BodyStyle.None
-                                && ShouldDeferCSharpEventOrDelegateSameLineAdvance(matchLine, lineOffset, pattern.Kind))
+                                && (lineOffset != patternStartOffset
+                                    ? ShouldDeferCSharpEventOrDelegateSameLineAdvance(matchLine, lineOffset, pattern.Kind)
+                                    : pattern.Kind == "event"
+                                        ? deferCSharpEventAtPatternStart ??=
+                                            ShouldDeferCSharpEventOrDelegateSameLineAdvance(matchLine, lineOffset, pattern.Kind)
+                                        : deferCSharpDelegateAtPatternStart ??=
+                                            ShouldDeferCSharpEventOrDelegateSameLineAdvance(matchLine, lineOffset, pattern.Kind)))
                             {
                                 break;
                             }
@@ -295,10 +312,16 @@ private static List<SymbolRecord> ExtractCore(
                                 || (lang == "csharp"
                                     && pattern.Kind == "property"
                                     && pattern.BodyStyle == BodyStyle.None
-                                    && !TryMatchAnyRecoverableCSharpPattern(
-                                        matchLine[lineOffset..],
-                                        insideEnumBody: false,
-                                        attributeParenDepth: 0)))
+                                    && !(lineOffset != patternStartOffset
+                                        ? TryMatchAnyRecoverableCSharpPattern(
+                                            matchLine[lineOffset..],
+                                            insideEnumBody: false,
+                                            attributeParenDepth: 0)
+                                        : recoverableCSharpPatternAtPatternStart ??=
+                                            TryMatchAnyRecoverableCSharpPattern(
+                                                matchLine[lineOffset..],
+                                                insideEnumBody: false,
+                                                attributeParenDepth: 0))))
                             {
                                 lineOffset = FindNextSameLineBraceStatementStart(matchLine, lineOffset + 1, lang);
                                 continue;

src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs

@@ -1246,6 +1246,7 @@ await EmitProgressNotificationAsync(
         // leaves prior indexed rows and trust metadata untouched even for rebuild requests.
         // rebuild破棄とFTS recoveryもwrite前scan barrier通過後まで遅延する。
         requestToken.ThrowIfCancellationRequested();
+        using var mmapBulkWrite = SqliteMmapBulkWriteGuard.Start(writer, useFtsBulkLoad);
         if (rebuild)
         {
             db.RepairIncompleteBatchReadiness();

tests/CodeIndex.Tests/DatabaseTests.cs

@@ -5957,6 +5957,27 @@ public void Constructor_ConfiguresConnectionPerformancePragmas()
             Assert.Equal(DbContext.DefaultMmapSizeBytes, ExecuteScalarLong("PRAGMA mmap_size"));
     }
 
+    [Fact]
+    public void SqliteMmapBulkWriteGuard_DisablesAndRestoresMappingAfterFailure()
+    {
+        if (!Environment.Is64BitProcess)
+            return;
+
+        var configuredMmapSizeBytes = ExecuteScalarLong("PRAGMA mmap_size");
+        if (configuredMmapSizeBytes == 0)
+            return;
+
+        Assert.Throws<InvalidOperationException>((Action)(() =>
+        {
+            using var guard = SqliteMmapBulkWriteGuard.Start(_writer, enabled: true);
+            Assert.NotNull(guard);
+            Assert.Equal(0L, ExecuteScalarLong("PRAGMA mmap_size"));
+            throw new InvalidOperationException("Simulated bulk-write failure.");
+        }));
+
+        Assert.Equal(configuredMmapSizeBytes, ExecuteScalarLong("PRAGMA mmap_size"));
+    }
+
     [Fact]
     public void Constructor_UsesSqlitePerformanceEnvironmentOverrides()
     {

tests/CodeIndex.Tests/PerformanceTests.cs

@@ -238,6 +238,28 @@ public void SymbolExtraction_CsharpHotPath_StaysWithinAllocationBudget()
         Assert.True(allocatedBytes < 18_000_000, $"Symbol extraction allocated {allocatedBytes:N0} bytes");
     }
 
+#if NET8_0
+    [Fact]
+#else
+    [Fact(Skip = PracticalBudgetTestTarget.SecondaryTargetSkipReason)]
+#endif
+    public void SymbolExtraction_CsharpSameLineRecoveryDecisions_StayWithinAllocationBudget()
+    {
+        const int propertyCount = 1_000;
+        var content = "public sealed class Fixture\n{\n"
+            + string.Join('\n', Enumerable.Range(0, propertyCount).Select(index =>
+                $"    public static Dictionary<string, List<(int Left, int Right)>> Property{index} {{ get; }} = new();"))
+            + "\n}";
+        var symbols = SymbolExtractor.Extract(1, "csharp", content);
+        Assert.Equal(propertyCount, symbols.Count(symbol => symbol.Kind == "property"));
+
+        var allocatedBytes = MeasureAllocatedBytes(() => SymbolExtractor.Extract(1, "csharp", content));
+
+        Assert.True(
+            allocatedBytes < 6_800_000,
+            $"C# same-line recovery extraction allocated {allocatedBytes:N0} bytes");
+    }
+
 #if NET8_0
     [Fact]
 #else