From 5fd64a148069c28557b7fa1c2fab6470ed0d9acc Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 10 Aug 2026 02:15:34 +0900 Subject: [PATCH 1/4] Fix filtered archive trust metadata (#5053) --- DEVELOPER_GUIDE.md | 4 + TESTING_GUIDE.md | 4 +- USER_GUIDE.md | 30 ++- changelog.d/unreleased/5053.fixed.md | 21 ++ .../ExportImportCommandRunner.ArchiveScope.cs | 250 +++++++++++++++++- .../ExportImportCommandRunner.Contracts.cs | 18 +- .../Cli/ExportImportCommandRunner.Import.cs | 7 + .../ExportImportCommandRunner.ImportOutput.cs | 10 +- ...ortImportCommandRunner.ImportValidation.cs | 51 +++- .../Cli/ExportImportCommandRunner.Manifest.cs | 6 + .../Cli/ExportImportCommandRunner.cs | 4 + .../Cli/ExportImportManifestCodec.cs | 78 +++++- .../ExportImportCommandRunnerTests.cs | 201 ++++++++++++++ 13 files changed, 668 insertions(+), 16 deletions(-) create mode 100644 changelog.d/unreleased/5053.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 634695fab..8aa2c36de 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -56,6 +56,8 @@ Development contracts: | Binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | Database compatibility across `cdidx` binary upgrades and downgrades is documented there. Keep that policy updated whenever readiness bits, `codeindex_meta` contract stamps, or rebuild requirements change. | | Fold backfill preview and recovery | `backfill-fold --dry-run`; `backfill-fold --checkpoint`; MCP `backfill_fold` with `dry_run: true` or `force: true` | Dry-run previews folded-key rows without mutating the DB or stamping FoldReady. `--checkpoint` explicitly preserves a snapshot even when CLI preflight finds no mutation, while the default completed no-op leaves no checkpoint artifacts. MCP accepts the same preview and can force rewriting all folded keys when an operator needs to recover from suspicious fold metadata or row state even though the stored version/fingerprint appears current. Non-dry-run row rewrites are resumable after interruption: completed row updates remain durable, and final FoldReady metadata is stamped only after verification succeeds. MCP responses include `progress.rows_done`, `progress.rows_total`, and `progress.fraction` so clients can report and retry long backfills. | +Portable archive trust is scope-aware. Unfiltered exports set `scope.represents_entire_source_database` and preserve completeness, indexed-HEAD, run, and unknown-extension metadata. Filtered exports normalize only the archive snapshot to `index_complete=false` with `partial_archive`, remove source-wide HEAD/run provenance, and reset unknown-extension summaries. Import repeats that normalization for legacy manifests without scope metadata, while current full-snapshot manifests preserve trust. + Checkpoint plan drift detection covers DB/WAL/SHM content changes and sidecar appearance or disappearance through the final pre-publication validation. The plan's `uncertainty` value records the remaining post-validation race; copied outputs are independently hash-verified against the plan before atomic publication. DB/WAL/SHM candidates must pass native regular-file type validation before hashing so Unix FIFOs cannot block planning. `metadata_policy` reports `owner_only_files_and_directories` on POSIX and the actually inherited `inherited_windows_acls` policy on Windows. A database payload whose output name collides with `manifest.txt`, including filesystem-equivalent casing, makes the plan not ready and is rejected before mutation. ## Filesystem Permissions @@ -3717,6 +3719,8 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま | binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | `cdidx` binary の upgrade / downgrade をまたぐ database compatibility を記載します。readiness bit、`codeindex_meta` contract stamp、rebuild requirement を変える場合は、この policy も更新してください。 | | Fold backfill の preview / recovery | `backfill-fold --dry-run`; `backfill-fold --checkpoint`; MCP `backfill_fold` の `dry_run: true` または `force: true` | dry-run は DB を変更せず FoldReady stamp も書かずに、rewrite 対象の folded-key row をプレビューします。CLI preflight でmutation不要と判断された場合でもsnapshotを明示的に保存するには `--checkpoint` を使います。既定の完了済みno-opはcheckpoint artifactを作りません。MCP も同じ preview を受け付け、stored version / fingerprint が current に見える場合でも suspicious な fold metadata や row state を復旧するため `force: true` を受け付けます。non-dry-run rewrite は中断後に resume でき、完了済み row update は durable に残り、最終 FoldReady metadata は verification 成功後にだけ stamp されます。MCP response は `progress.rows_done`、`progress.rows_total`、`progress.fraction` を含みます。 | +portable archive の trust は scope を考慮します。filter なし export は `scope.represents_entire_source_database` を設定し、completeness、indexed-HEAD、run、unknown-extension metadata を維持します。filter 済み export は archive snapshot だけを `index_complete=false` / `partial_archive` に正規化し、source 全体に対する HEAD / run provenance を削除して unknown-extension summary を reset します。scope metadata がない legacy manifest は import 時に同じ正規化を行い、現行の full-snapshot manifest は trust を維持します。 + checkpoint plan の drift 検出は、publish 前の最終検証までに起きる DB/WAL/SHM content の変更と sidecar の出現・消失を対象にします。plan の `uncertainty` 値は最終検証後に残る race を記録し、copy 済み output は atomic publish 前に plan の hash と個別に照合されます。DB/WAL/SHM 候補は hash 読み取り前に native regular-file type validation を通すため、Unix FIFO が plan 作成を停止させることはありません。`metadata_policy` は POSIX では `owner_only_files_and_directories`、Windows では実際に継承される `inherited_windows_acls` policy を報告します。database payload の output 名が filesystem 上で同一の大小文字を含め `manifest.txt` と衝突する場合、plan は not ready となり、変更前に拒否されます。 ## ファイルシステム権限 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 38c02c9d9..6f70d3b37 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -225,7 +225,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Inspect graph-section coverage must compare name and path/line resolution through the same persisted candidate ID, keep ambiguous overload and partial-family bundles isolated, assert independent total/returned/truncated metadata for references, callers, and callees (including empty sections), and replay a query-, page-size-, and generation-bound cursor across the smallest two-row page boundary. Put inbound callers in another file to prove the location path is only a locator, seed equal-rank same-line callees to pin the complete identity tie-breakers, reject page-size changes before candidate lookup, reject inspect cursors in another command, and verify the same envelopes in MCP `analyze_symbol`. Quiet-flag coverage in `ProgramCliTests.cs` reuses one seeded symbols database across text, NDJSON, and JSON-array modes and compares stdout with and without a trailing quiet alias, proving that quiet mode changes only informational stderr. Archive-import validation coverage reuses one pristine database export across read-only dry-run and check modes with distinct destinations. Rejection coverage copies one pristine export into manifest-count, database-hash, and user-version variants before mutating any ZIP so one corruption cannot contaminate another. - Archive success-path coverage seeds one metadata-rich database and shares its pristine export across scoped manifest inspection, a default import into a nonexistent destination, a copied legacy-manifest import, and a separate `--no-backup` replacement. Keep the default import and replacement as distinct CLI calls and destinations, and never mutate the pristine archive. + Archive success-path coverage seeds one metadata-rich database and shares its pristine export across scoped manifest inspection, a default import into a nonexistent destination, a copied legacy-manifest import, and a separate `--no-backup` replacement. Scope coverage must prove a filtered snapshot is incomplete with `partial_archive`, has no source-wide HEAD/run or unknown-extension trust, fails `status --check`, and leaves the source DB unchanged. Full-import coverage preserves those trust fields, while the copied scope-less manifest materializes the same conservative partial state. Keep the default import and replacement as distinct CLI calls and destinations, and never mutate the pristine archive. Doctor full-inventory coverage keeps composed filter selection, filtered summary counts, exact UTF-8 byte-budget boundaries, and structured overflow errors together in `ProgramRunnerTests`; license JSON remains a subprocess contract in `ProgramCliTests` so immediate-command dispatch and the published field names are both exercised. Ctags export JSON coverage reuses one seeded database for default and `--include-generated` variants, asserts the fixed skip-reason keys sum to `skipped_count`, and keeps the missing-`files.generated` degradation in a separate legacy-schema fixture. Dry-run JSON coverage for ambiguous `.h` files locks the bounded `language_detections` entries and their stable source/confidence codes without mutating the index. @@ -1223,7 +1223,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" inspect graph-section coverage では、name と path/line resolution が同じ persisted candidate ID を通ること、曖昧な overload と partial-family bundle が分離されること、references / callers / callees の独立した total / returned / truncated metadata（空 section を含む）を検証してください。inbound caller は別ファイルに置いて location path が locator にすぎないことを証明し、最小の2行 page 境界で query / page size / generation に束縛された cursor を再利用します。同順位かつ同じ行の callee で完全な identity tie-breaker を固定し、candidate lookup より前に page-size 変更を拒否し、別 command では inspect cursor を拒否してください。MCP `analyze_symbol` でも同じ envelope を確認します。 `ProgramCliTests.cs` の quiet flag coverage は1つの seeded symbols database を text、NDJSON、JSON array の各 mode で再利用し、末尾に quiet alias を付けた場合と付けない場合の stdout を比較して、quiet mode が informational stderr だけを変えることを固定します。 archive import validation coverage は、read-only な dry-run / check mode で1つの pristine database export を別々の destination から共有してください。拒否 coverage では、mutation 前に1つの pristine export を manifest-count、database-hash、user-version 用の3つの ZIP へコピーし、ある corruption が別 case を汚染しないようにします。 - archive success-path coverage は、metadata-rich な database を1回 seed し、その pristine export を scoped manifest inspection、存在しない destination への default import、コピーした legacy manifest の import、別 destination への `--no-backup` replacement で共有します。default import と replacement は別々の CLI 呼び出しと destination に保ち、pristine archive を直接変更しないでください。 + archive success-path coverage は、metadata-rich な database を1回 seed し、その pristine export を scoped manifest inspection、存在しない destination への default import、コピーした legacy manifest の import、別 destination への `--no-backup` replacement で共有します。scope coverage では filter 済み snapshot が `partial_archive` により incomplete となり、source 全体に対する HEAD / run / unknown-extension trust を持たず、`status --check` が失敗し、source DB が変更されないことを検証してください。full import はそれらの trust field を維持し、scope を除いた manifest のコピーは同じ保守的な partial state を materialize します。default import と replacement は別々の CLI 呼び出しと destination に保ち、pristine archive を直接変更しないでください。 doctor full-inventory coverage では、合成 filter の選択、filtered summary 件数、UTF-8 byte budget の exact boundary、structured overflow error を `ProgramRunnerTests` にまとめます。license JSON は `ProgramCliTests` の subprocess contract として、immediate-command dispatch と公開 field 名を同時に検証します。 ctags export JSON coverage は1つの seeded database を既定と `--include-generated` variant で再利用し、固定された skip-reason key の合計が `skipped_count` と一致することを検証します。`files.generated` がない場合の縮退は別の legacy-schema fixture に保ってください。 曖昧な `.h` に対する dry-run JSON coverage は、index を変更せず、上限付き `language_detections` entry と安定した判定元・信頼度 code を固定します。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 4fb40ff76..e6c0402d2 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -505,7 +505,15 @@ exclusion, and test filters then narrow that scope. The exported SQLite snapshot contains only the retained files and their dependent chunks, symbols, references, and diagnostics, and is vacuumed before packaging. JSON output and `manifest.json` include the requested scope, resolved project paths, and source -and exported file counts. An export without scope flags remains a full archive. +and exported file counts. The scope also reports +`represents_entire_source_database`. An export without scope flags remains a +full archive and preserves the source database's index completeness, +indexed-HEAD provenance, run telemetry, and unknown-extension summary. A +filtered archive is instead stamped `index_complete: false` with the stable +`partial_archive` reason, clears source-wide indexed-HEAD and run metadata, and +resets unknown-extension summaries so `status` cannot present the subset as a +fresh full index. This normalization applies only to the exported snapshot and +does not mutate the source database. Portable export refuses an existing destination by default; pass `--overwrite` only when replacing it is intentional. The archive is built in an owner-only sibling temporary file and atomically published, and POSIX archives are verified @@ -521,8 +529,11 @@ database. `--prune-paths` rewrites the imported `indexed_project_root` metadata to the import target project root. Imports targeting `.../.cdidx/codeindex.db` use the sibling project directory; other database paths fall back to the process current directory. `--dry-run` and its `--check` alias also compare an existing -destination DB with the validated archive without replacing it. JSON -`destination_delta.comparison` reports schema and count deltas plus bounded +destination DB with the validated archive without replacing it. JSON results +expose the normalized `index_complete`, `index_incomplete_reasons`, and `scope` +values. Archives with no scope metadata are treated conservatively as partial +during import; current unfiltered archives explicitly preserve full-snapshot +trust. JSON `destination_delta.comparison` reports schema and count deltas plus bounded file, symbol, reference-edge, chunk, and metadata records. Text fields in those records are represented by named SHA-256 and UTF-8 byte-length metadata rather than source content or paths. Use `--limit ` and `--offset ` to @@ -3999,7 +4010,13 @@ archive export では `--lang`、繰り返し指定できる `--path` / `--exclu さらに絞り込みます。出力する SQLite snapshot には残した file と、それに従属する chunk、symbol、reference、diagnostic だけを保持し、packaging 前に vacuum します。 JSON output と `manifest.json` には指定 scope、解決済み project path、元と出力後の -file count が含まれます。scope flag を指定しなければ従来どおり full archive です。 +file count が含まれます。scope には `represents_entire_source_database` も含まれます。 +scope flag を指定しない full archive は source database の index completeness、 +indexed-HEAD provenance、run telemetry、unknown-extension summary を維持します。一方、 +filter 済み archive は `index_complete: false` と stable reason `partial_archive` を記録し、 +source 全体に対する indexed-HEAD / run metadata を消去して unknown-extension summary を +reset するため、`status` が subset を fresh な full index として表示することはありません。 +この正規化は export snapshot だけに適用され、source database は変更しません。 portable export は既存 destination を既定で拒否します。意図して置き換える場合だけ `--overwrite` を指定してください。archive は owner-only の sibling temporary file に 構築して atomic に publish し、POSIX では mode `0600` であることも検証します。 @@ -4014,7 +4031,10 @@ SQLite file が CodeIndex DB であることを検証してから destination da `.../.cdidx/codeindex.db` を import 先にした場合は sibling の project directory を使い、 それ以外の database path では process current directory に fallback します。 `--dry-run` と alias の `--check` は置換せず、既存 destination DB と検証済み archive を -比較します。JSON の `destination_delta.comparison` には schema / count delta と、 +比較します。JSON result は正規化後の `index_complete`、`index_incomplete_reasons`、 +`scope` を公開します。scope metadata がない archive は import 時に保守的に partial と +扱い、現行の filter なし archive だけが full snapshot の trust を明示的に維持します。 +JSON の `destination_delta.comparison` には schema / count delta と、 file、symbol、reference edge、chunk、metadata の bounded record が含まれます。 これらの record の text field は source content や path そのものではなく、名前付きの SHA-256 と UTF-8 byte length metadata として表現されます。record の paging には diff --git a/changelog.d/unreleased/5053.fixed.md b/changelog.d/unreleased/5053.fixed.md new file mode 100644 index 000000000..c40215228 --- /dev/null +++ b/changelog.d/unreleased/5053.fixed.md @@ -0,0 +1,21 @@ +--- +category: fixed +issues: + - 5053 +affected: + - src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs + - src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs + - src/CodeIndex/Cli/ExportImportManifestCodec.cs + - tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Filtered portable archives now report conservative trust metadata (#5053)** — Scoped exports and legacy scope-less imports are marked incomplete with `partial_archive`, clear source-wide HEAD/run provenance, and reset unknown-extension summaries, while full unfiltered archives preserve their trust metadata. + +## 日本語 + +- **filter 済み portable archive が保守的な trust metadata を報告するようになりました (#5053)** — scope 付き export と scope 情報がない legacy import は `partial_archive` により incomplete とし、source 全体に対する HEAD / run provenance を削除して unknown-extension summary を reset します。filter なしの full archive は従来の trust metadata を維持します。 diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs index 319452ecc..8004380b8 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs @@ -50,7 +50,8 @@ private static ArchiveExportScopeResult ApplyArchiveScope( options.ExcludeTests, projectPathPatterns, sourceFileCount, - sourceFileCount); + sourceFileCount, + RepresentsEntireSourceDatabase: true); } using (var foreignKeys = connection.CreateCommand()) @@ -131,6 +132,7 @@ WHERE reference_id NOT IN (SELECT id FROM symbol_references) """; pruneCommand.ExecuteNonQuery(); DbWriter.RebuildRetainedReferenceGraph(connection, transaction, cancellationToken); + ApplyPartialArchiveTrustMetadata(connection, transaction, cancellationToken); transaction.Commit(); } @@ -160,7 +162,251 @@ WHERE reference_id NOT IN (SELECT id FROM symbol_references) options.ExcludeTests, projectPathPatterns, sourceFileCount, - exportedFileCount); + exportedFileCount, + RepresentsEntireSourceDatabase: false); + } + + private static void ApplyImportedArchiveTrustMetadata( + string databasePath, + ExportManifest manifest, + CancellationToken cancellationToken) + { + if (ManifestRepresentsEntireSourceDatabase(manifest)) + return; + + cancellationToken.ThrowIfCancellationRequested(); + using var connection = new SqliteConnection(CreateUnpooledConnectionString(databasePath)); + connection.Open(); + if (!ArchiveTrustMetadataRequiresNormalization(connection)) + return; + + using var transaction = connection.BeginTransaction(); + ApplyPartialArchiveTrustMetadata(connection, transaction, cancellationToken); + transaction.Commit(); + } + + private static bool ArchiveTrustMetadataRequiresNormalization(SqliteConnection connection) + { + if (!string.Equals( + ReadMetaString(connection, DbContext.IndexCompletenessMetaKey), + "incomplete", + StringComparison.Ordinal) + || !HasPartialArchiveIncompleteReason( + ReadMetaString(connection, DbContext.IndexIncompleteReasonsMetaKey)) + || ReadMetaString(connection, DbContext.UnknownExtensionFileCountMetaKey) != "0" + || ReadMetaString(connection, DbContext.UnknownExtensionFilePathsMetaKey) != "[]" + || !bool.TryParse( + ReadMetaString(connection, DbContext.UnknownExtensionFilesTruncatedMetaKey), + out var unknownFilesTruncated) + || unknownFilesTruncated + || ReadMetaString(connection, DbContext.UnknownExtensionExtensionCountsMetaKey) != "{}" + || ReadMetaString(connection, DbContext.UnknownExtensionCategoryCountsMetaKey) != "{}" + || ReadMetaString(connection, DbContext.UnknownExtensionGroupsMetaKey) != "[]") + { + return true; + } + + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT COUNT(*) + FROM codeindex_meta + WHERE key GLOB 'indexed_head_*' + OR key GLOB 'last_index_run_*' + OR key GLOB 'last_failed_index_run_*' + OR key IN ( + 'commit_scoped_fresh_head_sha', + 'last_full_scan_elapsed_ms', + 'last_workspace_freshened_at') + """; + return Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture) != 0; + } + + private static bool HasPartialArchiveIncompleteReason(string? rawReasons) + { + if (string.IsNullOrWhiteSpace(rawReasons) + || Encoding.UTF8.GetByteCount(rawReasons) > MaxImportManifestBytes) + { + return false; + } + + try + { + using var document = JsonDocument.Parse( + rawReasons, + new JsonDocumentOptions { MaxDepth = 4 }); + return document.RootElement.ValueKind == JsonValueKind.Array + && document.RootElement.EnumerateArray().Any(item => + item.ValueKind == JsonValueKind.String + && string.Equals( + item.GetString(), + PartialArchiveIncompleteReason, + StringComparison.Ordinal)); + } + catch (JsonException) + { + return false; + } + } + + private static ExportManifest NormalizeImportedArchiveTrustMetadata(ExportManifest manifest) + { + if (ManifestRepresentsEntireSourceDatabase(manifest)) + return manifest; + + var reasons = new List(MaxArchiveIncompleteReasons); + var totalChars = 0; + foreach (var reason in manifest.IndexIncompleteReasons ?? []) + { + if (string.Equals(reason, PartialArchiveIncompleteReason, StringComparison.Ordinal)) + continue; + if (reasons.Count >= MaxArchiveIncompleteReasons - 1 + || totalChars + reason.Length + PartialArchiveIncompleteReason.Length + > MaxArchiveIncompleteReasonsTotalChars) + { + break; + } + reasons.Add(reason); + totalChars += reason.Length; + } + reasons.Add(PartialArchiveIncompleteReason); + return manifest with + { + IndexedHeadSha = null, + IndexedHeadBranch = null, + IndexedHeadTimestamp = null, + UnknownExtensionFileCount = 0, + UnknownExtensionFiles = null, + UnknownExtensionFilesTruncated = false, + UnknownExtensionFileSampleCount = 0, + UnknownExtensionFileSampleTruncated = false, + IndexComplete = false, + IndexIncompleteReasons = reasons.ToArray(), + }; + } + + private static bool ManifestRepresentsEntireSourceDatabase(ExportManifest manifest) + => manifest.Scope?.RepresentsEntireSourceDatabase == true; + + private static void ApplyPartialArchiveTrustMetadata( + SqliteConnection connection, + SqliteTransaction transaction, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using (var ensureMeta = connection.CreateCommand()) + { + ensureMeta.Transaction = transaction; + ensureMeta.CommandText = """ + CREATE TABLE IF NOT EXISTS codeindex_meta ( + key TEXT PRIMARY KEY NOT NULL, + value TEXT + ) + """; + ensureMeta.ExecuteNonQuery(); + } + var incompleteReasonsJson = BuildPartialArchiveIncompleteReasonsJson(connection, transaction); + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = """ + DELETE FROM codeindex_meta + WHERE key GLOB 'indexed_head_*' + OR key GLOB 'last_index_run_*' + OR key GLOB 'last_failed_index_run_*' + OR key IN ( + 'commit_scoped_fresh_head_sha', + 'last_full_scan_elapsed_ms', + 'last_workspace_freshened_at'); + + INSERT INTO codeindex_meta(key, value) + VALUES (@indexCompletenessKey, 'incomplete') + ON CONFLICT(key) DO UPDATE SET value = excluded.value; + + INSERT INTO codeindex_meta(key, value) + VALUES (@indexIncompleteReasonsKey, @indexIncompleteReasons) + ON CONFLICT(key) DO UPDATE SET value = excluded.value; + + INSERT INTO codeindex_meta(key, value) + VALUES (@unknownExtensionFileCountKey, '0') + ON CONFLICT(key) DO UPDATE SET value = excluded.value; + + INSERT INTO codeindex_meta(key, value) + VALUES (@unknownExtensionFilePathsKey, '[]') + ON CONFLICT(key) DO UPDATE SET value = excluded.value; + + INSERT INTO codeindex_meta(key, value) + VALUES (@unknownExtensionFilesTruncatedKey, 'False') + ON CONFLICT(key) DO UPDATE SET value = excluded.value; + + INSERT INTO codeindex_meta(key, value) + VALUES (@unknownExtensionExtensionCountsKey, '{}') + ON CONFLICT(key) DO UPDATE SET value = excluded.value; + + INSERT INTO codeindex_meta(key, value) + VALUES (@unknownExtensionCategoryCountsKey, '{}') + ON CONFLICT(key) DO UPDATE SET value = excluded.value; + + INSERT INTO codeindex_meta(key, value) + VALUES (@unknownExtensionGroupsKey, '[]') + ON CONFLICT(key) DO UPDATE SET value = excluded.value; + """; + SqliteCommandPolicy.Add(command, "@indexCompletenessKey", DbContext.IndexCompletenessMetaKey); + SqliteCommandPolicy.Add(command, "@indexIncompleteReasonsKey", DbContext.IndexIncompleteReasonsMetaKey); + SqliteCommandPolicy.Add(command, "@indexIncompleteReasons", incompleteReasonsJson); + SqliteCommandPolicy.Add(command, "@unknownExtensionFileCountKey", DbContext.UnknownExtensionFileCountMetaKey); + SqliteCommandPolicy.Add(command, "@unknownExtensionFilePathsKey", DbContext.UnknownExtensionFilePathsMetaKey); + SqliteCommandPolicy.Add(command, "@unknownExtensionFilesTruncatedKey", DbContext.UnknownExtensionFilesTruncatedMetaKey); + SqliteCommandPolicy.Add(command, "@unknownExtensionExtensionCountsKey", DbContext.UnknownExtensionExtensionCountsMetaKey); + SqliteCommandPolicy.Add(command, "@unknownExtensionCategoryCountsKey", DbContext.UnknownExtensionCategoryCountsMetaKey); + SqliteCommandPolicy.Add(command, "@unknownExtensionGroupsKey", DbContext.UnknownExtensionGroupsMetaKey); + command.ExecuteNonQuery(); + cancellationToken.ThrowIfCancellationRequested(); + } + + private static string BuildPartialArchiveIncompleteReasonsJson( + SqliteConnection connection, + SqliteTransaction transaction) + { + var reasons = new List(MaxArchiveIncompleteReasons); + var seen = new HashSet(StringComparer.Ordinal); + var rawReasons = ReadMetaString(connection, DbContext.IndexIncompleteReasonsMetaKey, transaction); + if (!string.IsNullOrWhiteSpace(rawReasons) + && Encoding.UTF8.GetByteCount(rawReasons) <= MaxImportManifestBytes) + { + try + { + using var document = JsonDocument.Parse( + rawReasons, + new JsonDocumentOptions { MaxDepth = 4 }); + if (document.RootElement.ValueKind == JsonValueKind.Array) + { + var totalChars = 0; + foreach (var item in document.RootElement.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String || reasons.Count >= MaxArchiveIncompleteReasons - 1) + break; + var reason = item.GetString(); + if (string.IsNullOrWhiteSpace(reason) + || reason.Length > MaxArchiveIncompleteReasonChars + || totalChars + reason.Length + PartialArchiveIncompleteReason.Length + > MaxArchiveIncompleteReasonsTotalChars + || !seen.Add(reason)) + { + continue; + } + reasons.Add(reason); + totalChars += reason.Length; + } + } + } + catch (JsonException) + { + // Invalid legacy metadata is replaced by the stable partial-archive reason. + } + } + + if (seen.Add(PartialArchiveIncompleteReason)) + reasons.Add(PartialArchiveIncompleteReason); + return JsonSerializer.Serialize(reasons); } private static bool TryValidateArchiveScopeValues( diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.Contracts.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.Contracts.cs index df2f9e866..23c04e205 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.Contracts.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.Contracts.cs @@ -163,6 +163,10 @@ internal sealed record ExportManifest( int? UnknownExtensionFileSampleLimit = null, [property: JsonPropertyName("unknown_extension_file_sample_truncated")] bool? UnknownExtensionFileSampleTruncated = null, + [property: JsonPropertyName("index_complete")] + bool? IndexComplete = null, + [property: JsonPropertyName("index_incomplete_reasons")] + string[]? IndexIncompleteReasons = null, [property: JsonPropertyName("scope")] ArchiveExportScopeResult? Scope = null); internal sealed record ExportImportErrorResult( @@ -214,7 +218,10 @@ internal sealed record ImportDryRunResult( [property: JsonPropertyName("unknown_extension_file_path_limit")] int? UnknownExtensionFilePathLimit = null, [property: JsonPropertyName("unknown_extension_file_sample_count")] int? UnknownExtensionFileSampleCount = null, [property: JsonPropertyName("unknown_extension_file_sample_limit")] int? UnknownExtensionFileSampleLimit = null, - [property: JsonPropertyName("unknown_extension_file_sample_truncated")] bool? UnknownExtensionFileSampleTruncated = null); + [property: JsonPropertyName("unknown_extension_file_sample_truncated")] bool? UnknownExtensionFileSampleTruncated = null, + [property: JsonPropertyName("index_complete")] bool? IndexComplete = null, + [property: JsonPropertyName("index_incomplete_reasons")] string[]? IndexIncompleteReasons = null, + [property: JsonPropertyName("scope")] ArchiveExportScopeResult? Scope = null); internal sealed record ImportDestinationDeltaResult( [property: JsonPropertyName("destination_exists")] bool DestinationExists, [property: JsonPropertyName("comparable")] bool Comparable, @@ -257,7 +264,9 @@ internal sealed record ArchiveExportScopeResult( [property: JsonPropertyName("exclude_tests")] bool ExcludeTests, [property: JsonPropertyName("resolved_project_path")] IReadOnlyList ResolvedProjectPathPatterns, [property: JsonPropertyName("source_file_count")] long SourceFileCount, - [property: JsonPropertyName("exported_file_count")] long ExportedFileCount); + [property: JsonPropertyName("exported_file_count")] long ExportedFileCount, + [property: JsonPropertyName("represents_entire_source_database")] + bool? RepresentsEntireSourceDatabase = null); private sealed record CtagsExportOptions( string? Lang, IReadOnlyList PathPatterns, @@ -306,7 +315,10 @@ internal sealed record ImportResult( [property: JsonPropertyName("unknown_extension_file_path_limit")] int? UnknownExtensionFilePathLimit = null, [property: JsonPropertyName("unknown_extension_file_sample_count")] int? UnknownExtensionFileSampleCount = null, [property: JsonPropertyName("unknown_extension_file_sample_limit")] int? UnknownExtensionFileSampleLimit = null, - [property: JsonPropertyName("unknown_extension_file_sample_truncated")] bool? UnknownExtensionFileSampleTruncated = null); + [property: JsonPropertyName("unknown_extension_file_sample_truncated")] bool? UnknownExtensionFileSampleTruncated = null, + [property: JsonPropertyName("index_complete")] bool? IndexComplete = null, + [property: JsonPropertyName("index_incomplete_reasons")] string[]? IndexIncompleteReasons = null, + [property: JsonPropertyName("scope")] ArchiveExportScopeResult? Scope = null); private sealed class ImportReplacementException : IOException { diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.Import.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.Import.cs index 669b83a23..9ce82f49e 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.Import.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.Import.cs @@ -115,6 +115,13 @@ public static int RunImport(string[] args, JsonSerializerOptions jsonOptions, Ca AddImportValidationPhase(validationPhases, PhaseSqliteValidate); SqliteConnection.ClearAllPools(); + ApplyImportedArchiveTrustMetadata( + tempPath, + importedManifest ?? throw new InvalidDataException("archive manifest was not loaded"), + cancellationToken); + importedManifest = NormalizeImportedArchiveTrustMetadata(importedManifest); + SqliteConnection.ClearAllPools(); + if (prunePaths) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.ImportOutput.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.ImportOutput.cs index 521b7f863..d96a23fff 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.ImportOutput.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.ImportOutput.cs @@ -48,7 +48,10 @@ private static int WriteImportDryRunResult( UnknownExtensionFilePathLimit: manifest.UnknownExtensionFilePathLimit, UnknownExtensionFileSampleCount: manifest.UnknownExtensionFileSampleCount, UnknownExtensionFileSampleLimit: manifest.UnknownExtensionFileSampleLimit, - UnknownExtensionFileSampleTruncated: manifest.UnknownExtensionFileSampleTruncated), + UnknownExtensionFileSampleTruncated: manifest.UnknownExtensionFileSampleTruncated, + IndexComplete: manifest.IndexComplete, + IndexIncompleteReasons: manifest.IndexIncompleteReasons, + Scope: manifest.Scope), CliJsonSerializerContextFactory.Create(jsonOptions).ImportDryRunResult)); } else @@ -92,7 +95,10 @@ private static int WriteImportResult( UnknownExtensionFilePathLimit: manifest.UnknownExtensionFilePathLimit, UnknownExtensionFileSampleCount: manifest.UnknownExtensionFileSampleCount, UnknownExtensionFileSampleLimit: manifest.UnknownExtensionFileSampleLimit, - UnknownExtensionFileSampleTruncated: manifest.UnknownExtensionFileSampleTruncated), + UnknownExtensionFileSampleTruncated: manifest.UnknownExtensionFileSampleTruncated, + IndexComplete: manifest.IndexComplete, + IndexIncompleteReasons: manifest.IndexIncompleteReasons, + Scope: manifest.Scope), jsonOptions)); } else diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.ImportValidation.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.ImportValidation.cs index 37a811389..8c6d83d68 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.ImportValidation.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.ImportValidation.cs @@ -211,9 +211,13 @@ private static long ReadTableCount(SqliteConnection connection, string tableName return count; } - private static string? ReadMetaString(SqliteConnection connection, string key) + private static string? ReadMetaString( + SqliteConnection connection, + string key, + SqliteTransaction? transaction = null) { using var cmd = connection.CreateCommand(); + cmd.Transaction = transaction; cmd.CommandText = "SELECT value FROM codeindex_meta WHERE key = @key LIMIT 1"; SqliteCommandPolicy.Add(cmd, "@key", key); return cmd.ExecuteScalar() as string; @@ -241,6 +245,51 @@ private static long ReadTableCount(SqliteConnection connection, string tableName return bool.TryParse(value, out var parsed) ? parsed : null; } + private static string[]? ReadArchiveIncompleteReasons(SqliteConnection connection) + { + var rawReasons = ReadMetaString(connection, DbContext.IndexIncompleteReasonsMetaKey); + if (string.IsNullOrWhiteSpace(rawReasons) + || Encoding.UTF8.GetByteCount(rawReasons) > MaxImportManifestBytes) + { + return null; + } + + try + { + using var document = JsonDocument.Parse( + rawReasons, + new JsonDocumentOptions { MaxDepth = 4 }); + if (document.RootElement.ValueKind != JsonValueKind.Array) + return null; + + var reasons = new List(MaxArchiveIncompleteReasons); + var seen = new HashSet(StringComparer.Ordinal); + var totalChars = 0; + foreach (var item in document.RootElement.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String || reasons.Count >= MaxArchiveIncompleteReasons) + return null; + var reason = item.GetString(); + if (string.IsNullOrWhiteSpace(reason) + || reason.Length > MaxArchiveIncompleteReasonChars) + { + return null; + } + if (!seen.Add(reason)) + continue; + if (totalChars + reason.Length > MaxArchiveIncompleteReasonsTotalChars) + return null; + reasons.Add(reason); + totalChars += reason.Length; + } + return reasons.ToArray(); + } + catch (JsonException) + { + return null; + } + } + private readonly record struct UnknownExtensionFileSample(string[]? Files, int? Count, int? Limit, bool? Truncated); private static UnknownExtensionFileSample ReadUnknownExtensionFileSample(SqliteConnection connection) diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs index a964d2549..76a95e72e 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs @@ -25,6 +25,8 @@ private static ExportManifest BuildManifest( var projectRoot = ReadMetaString(connection, DbContext.IndexedProjectRootMetaKey); var indexedHead = ReadMetaString(connection, DbContext.IndexedHeadShaMetaKey); var unknownExtensionFiles = ReadUnknownExtensionFileSample(connection); + var indexCompleteness = ReadMetaString(connection, DbContext.IndexCompletenessMetaKey); + var indexIncompleteReasons = ReadArchiveIncompleteReasons(connection); cancellationToken.ThrowIfCancellationRequested(); return new ExportManifest( "1", @@ -54,6 +56,10 @@ private static ExportManifest BuildManifest( UnknownExtensionFileSampleCount: unknownExtensionFiles.Count, UnknownExtensionFileSampleLimit: unknownExtensionFiles.Limit, UnknownExtensionFileSampleTruncated: unknownExtensionFiles.Truncated, + IndexComplete: indexCompleteness == null + ? null + : string.Equals(indexCompleteness, "complete", StringComparison.Ordinal), + IndexIncompleteReasons: indexIncompleteReasons, Scope: scope); } diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.cs index dcd9377f3..410a5f778 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.cs @@ -30,6 +30,10 @@ internal static partial class ExportImportCommandRunner internal const int MaxArchiveScopeValues = 64; internal const int MaxArchiveScopeValueChars = 4096; internal const int MaxArchiveScopeTotalChars = 32 * 1024; + internal const int MaxArchiveIncompleteReasons = 64; + internal const int MaxArchiveIncompleteReasonChars = 128; + internal const int MaxArchiveIncompleteReasonsTotalChars = 4096; + internal const string PartialArchiveIncompleteReason = "partial_archive"; private static readonly DateTimeOffset DeterministicZipTimestamp = new(1980, 1, 1, 0, 0, 0, TimeSpan.Zero); private const string ExportCommandName = "export"; private const string ImportCommandName = "import"; diff --git a/src/CodeIndex/Cli/ExportImportManifestCodec.cs b/src/CodeIndex/Cli/ExportImportManifestCodec.cs index 5231913dd..2a40451de 100644 --- a/src/CodeIndex/Cli/ExportImportManifestCodec.cs +++ b/src/CodeIndex/Cli/ExportImportManifestCodec.cs @@ -171,9 +171,15 @@ internal static bool TryValidateHeader(ExportImportCommandRunner.ExportManifest } } - if (manifest.Scope != null && !TryValidateScope(manifest.Scope, out message)) + if (!TryValidateIncompleteReasons(manifest, out message)) return false; + if (manifest.Scope != null) + { + if (!TryValidateScope(manifest.Scope, out message)) + return false; + } + message = string.Empty; return true; } @@ -198,6 +204,24 @@ private static bool TryValidateScope( return false; } + var hasSelection = !string.IsNullOrWhiteSpace(scope.Lang) + || scope.PathPatterns.Count > 0 + || scope.ExcludePathPatterns.Count > 0 + || scope.Projects.Count > 0 + || !string.IsNullOrWhiteSpace(scope.Solution) + || scope.ExcludeTests; + if (scope.Scoped != hasSelection) + { + message = "scope.scoped does not match the recorded selection metadata"; + return false; + } + if (scope.RepresentsEntireSourceDatabase == true + && (scope.Scoped || scope.SourceFileCount != scope.ExportedFileCount)) + { + message = "scope cannot represent the entire source database when filters or omitted files are recorded"; + return false; + } + var values = scope.PathPatterns .Concat(scope.ExcludePathPatterns) .Concat(scope.Projects) @@ -236,6 +260,58 @@ private static bool TryValidateScope( return true; } + private static bool TryValidateIncompleteReasons( + ExportImportCommandRunner.ExportManifest manifest, + out string message) + { + if (manifest.IndexIncompleteReasons is not { } reasons) + { + message = string.Empty; + return true; + } + if (reasons.Length > ExportImportCommandRunner.MaxArchiveIncompleteReasons) + { + message = $"index_incomplete_reasons exceeds the manifest limit of {ExportImportCommandRunner.MaxArchiveIncompleteReasons}"; + return false; + } + + var totalChars = 0; + var seen = new HashSet(StringComparer.Ordinal); + foreach (var reason in reasons) + { + if (string.IsNullOrWhiteSpace(reason)) + { + message = "index_incomplete_reasons contains an empty reason"; + return false; + } + if (reason.Length > ExportImportCommandRunner.MaxArchiveIncompleteReasonChars) + { + message = $"index_incomplete_reasons contains a reason longer than {ExportImportCommandRunner.MaxArchiveIncompleteReasonChars} characters"; + return false; + } + totalChars += reason.Length; + if (totalChars > ExportImportCommandRunner.MaxArchiveIncompleteReasonsTotalChars) + { + message = $"index_incomplete_reasons exceeds {ExportImportCommandRunner.MaxArchiveIncompleteReasonsTotalChars} total characters"; + return false; + } + if (!seen.Add(reason)) + { + message = "index_incomplete_reasons contains a duplicate reason"; + return false; + } + } + + if (manifest.IndexComplete == true && reasons.Length > 0) + { + message = "index_complete cannot be true when index_incomplete_reasons are present"; + return false; + } + + message = string.Empty; + return true; + } + internal static JsonSerializerOptions CreateImportManifestJsonOptions(JsonSerializerOptions jsonOptions) => new(jsonOptions) { MaxDepth = ExportImportCommandRunner.MaxImportManifestJsonDepth }; diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index a8581680c..e8578fb65 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -3,6 +3,7 @@ using System.Security.Cryptography; using System.Text; using System.Text.Json; +using System.Text.Json.Nodes; using CodeIndex.Cli; using CodeIndex.Database; using Microsoft.Data.Sqlite; @@ -979,6 +980,7 @@ public void RunExportArchive_AppliesProjectPathLanguageAndTestScope_Issue4714() TestProjectHelper.InsertIndexedFile(dbPath, "src/Other/Other.cs", "csharp", "public class OtherType { }"); TestProjectHelper.InsertIndexedFile(dbPath, "src/App/tool.py", "python", "def tool(): pass"); TestProjectHelper.InsertIndexedFile(dbPath, "src/App/tests/AppTests.cs", "csharp", "public class AppTests { }"); + SetArchiveTrustMetadata(dbPath, 5); var archivePath = Path.Combine(projectRoot, "scoped.cdidx.zip"); var jsonOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower }; @@ -1004,6 +1006,18 @@ public void RunExportArchive_AppliesProjectPathLanguageAndTestScope_Issue4714() Assert.Equal(5, scope.GetProperty("source_file_count").GetInt64()); Assert.Equal(2, scope.GetProperty("exported_file_count").GetInt64()); Assert.Equal("src/App/*", scope.GetProperty("resolved_project_path")[0].GetString()); + Assert.False(scope.GetProperty("represents_entire_source_database").GetBoolean()); + var resultManifest = result.RootElement.GetProperty("manifest"); + Assert.False(resultManifest.GetProperty("index_complete").GetBoolean()); + Assert.Contains( + resultManifest.GetProperty("index_incomplete_reasons").EnumerateArray(), + reason => reason.GetString() == ExportImportCommandRunner.PartialArchiveIncompleteReason); + Assert.Equal(0, resultManifest.GetProperty("unknown_extension_file_count").GetInt64()); + Assert.Equal(string.Empty, ReadMetaValue(dbPath, DbContext.IndexIncompleteReasonsMetaKey)); + Assert.Equal("source-head", ReadMetaValue(dbPath, DbContext.IndexedHeadShaMetaKey)); + Assert.Equal("source-head", ReadMetaValue(dbPath, DbContext.CommitScopedFreshHeadShaMetaKey)); + Assert.Equal("5", ReadMetaValue(dbPath, DbContext.LastIndexRunFilesScannedMetaKey)); + Assert.Equal("2", ReadMetaValue(dbPath, DbContext.UnknownExtensionFileCountMetaKey)); var extractedDb = Path.Combine(projectRoot, "scoped.db"); using (var archive = ZipFile.OpenRead(archivePath)) @@ -1026,6 +1040,33 @@ public void RunExportArchive_AppliesProjectPathLanguageAndTestScope_Issue4714() Assert.False(reader.Read()); } + Assert.Equal("incomplete", ReadMetaValue(extractedDb, DbContext.IndexCompletenessMetaKey)); + using (var incompleteReasons = JsonDocument.Parse( + ReadMetaValue(extractedDb, DbContext.IndexIncompleteReasonsMetaKey)!)) + { + Assert.Contains( + incompleteReasons.RootElement.EnumerateArray(), + reason => reason.GetString() == ExportImportCommandRunner.PartialArchiveIncompleteReason); + } + Assert.Null(ReadMetaValue(extractedDb, DbContext.IndexedHeadShaMetaKey)); + Assert.Null(ReadMetaValue(extractedDb, DbContext.CommitScopedFreshHeadShaMetaKey)); + Assert.Null(ReadMetaValue(extractedDb, DbContext.LastIndexRunFilesScannedMetaKey)); + Assert.Equal("0", ReadMetaValue(extractedDb, DbContext.UnknownExtensionFileCountMetaKey)); + Assert.Equal("[]", ReadMetaValue(extractedDb, DbContext.UnknownExtensionFilePathsMetaKey)); + AssertPartialArchiveStatus(extractedDb); + + var importedDb = Path.Combine(projectRoot, "imported", "codeindex.db"); + var (importExitCode, _, importStderr) = ConsoleCapture.Capture(() => + ExportImportCommandRunner.RunImport( + [archivePath, "--db", importedDb, "--json"], + jsonOptions)); + Assert.Equal(CommandExitCodes.Success, importExitCode); + Assert.Equal(string.Empty, importStderr); + Assert.Equal( + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(extractedDb))), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(importedDb)))); + AssertPartialArchiveStatus(importedDb); + using var manifest = ZipFile.OpenRead(archivePath); using var manifestStream = manifest.GetEntry("manifest.json")!.Open(); using var manifestDocument = JsonDocument.Parse(manifestStream); @@ -1038,6 +1079,78 @@ public void RunExportArchive_AppliesProjectPathLanguageAndTestScope_Issue4714() } } + [Fact] + public void RunImport_FullAndLegacyArchivesMaterializeConservativeTrust_Issue5053() + { + var projectRoot = TestProjectHelper.CreateTempProject("import_archive_trust_scope"); + try + { + var sourceDbPath = TestProjectHelper.CreateProjectDb(Path.Combine(projectRoot, "source")); + TestProjectHelper.InsertIndexedFile( + sourceDbPath, + "src/App.cs", + "csharp", + "public class App { public void Run() { } }"); + SetArchiveTrustMetadata(sourceDbPath, 1); + var pristineArchivePath = ExportArchive(projectRoot, sourceDbPath); + var jsonOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower }; + + var fullImportDbPath = Path.Combine(projectRoot, "full", "codeindex.db"); + var (fullExitCode, fullStdout, fullStderr) = ConsoleCapture.Capture(() => + ExportImportCommandRunner.RunImport( + [pristineArchivePath, "--db", fullImportDbPath, "--json"], + jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, fullExitCode); + Assert.Equal(string.Empty, fullStderr); + using (var fullResult = JsonDocument.Parse(fullStdout)) + { + Assert.True(fullResult.RootElement.GetProperty("index_complete").GetBoolean()); + Assert.True(fullResult.RootElement + .GetProperty("scope") + .GetProperty("represents_entire_source_database") + .GetBoolean()); + Assert.Equal(2, fullResult.RootElement.GetProperty("unknown_extension_file_count").GetInt64()); + } + Assert.Equal("complete", ReadMetaValue(fullImportDbPath, DbContext.IndexCompletenessMetaKey)); + Assert.Equal("source-head", ReadMetaValue(fullImportDbPath, DbContext.IndexedHeadShaMetaKey)); + Assert.Equal("1", ReadMetaValue(fullImportDbPath, DbContext.LastIndexRunFilesScannedMetaKey)); + Assert.Equal("2", ReadMetaValue(fullImportDbPath, DbContext.UnknownExtensionFileCountMetaKey)); + + var legacyArchivePath = CopyArchiveWithoutTrustScopeMarker(projectRoot, pristineArchivePath); + var legacyImportDbPath = Path.Combine(projectRoot, "legacy", "codeindex.db"); + var (legacyExitCode, legacyStdout, legacyStderr) = ConsoleCapture.Capture(() => + ExportImportCommandRunner.RunImport( + [legacyArchivePath, "--db", legacyImportDbPath, "--json"], + jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, legacyExitCode); + Assert.Equal(string.Empty, legacyStderr); + using (var legacyResult = JsonDocument.Parse(legacyStdout)) + { + Assert.False(legacyResult.RootElement.GetProperty("index_complete").GetBoolean()); + Assert.Contains( + legacyResult.RootElement.GetProperty("index_incomplete_reasons").EnumerateArray(), + reason => reason.GetString() == ExportImportCommandRunner.PartialArchiveIncompleteReason); + var legacyScope = legacyResult.RootElement.GetProperty("scope"); + Assert.True( + !legacyScope.TryGetProperty("represents_entire_source_database", out var wholeSource) + || wholeSource.ValueKind == JsonValueKind.Null); + Assert.Equal(0, legacyResult.RootElement.GetProperty("unknown_extension_file_count").GetInt64()); + } + Assert.Equal("incomplete", ReadMetaValue(legacyImportDbPath, DbContext.IndexCompletenessMetaKey)); + Assert.Null(ReadMetaValue(legacyImportDbPath, DbContext.IndexedHeadShaMetaKey)); + Assert.Null(ReadMetaValue(legacyImportDbPath, DbContext.CommitScopedFreshHeadShaMetaKey)); + Assert.Null(ReadMetaValue(legacyImportDbPath, DbContext.LastIndexRunFilesScannedMetaKey)); + Assert.Equal("0", ReadMetaValue(legacyImportDbPath, DbContext.UnknownExtensionFileCountMetaKey)); + AssertPartialArchiveStatus(legacyImportDbPath); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunExportArchive_MigratesLegacySnapshotBeforeApplyingScope_Issue4714() { @@ -2158,6 +2271,94 @@ private static string ExportArchive(string projectRoot, string dbPath) return archivePath; } + private static string CopyArchiveWithoutTrustScopeMarker(string projectRoot, string archivePath) + { + var legacyArchivePath = Path.Combine(projectRoot, $"legacy-{Guid.NewGuid():N}.cdidx.zip"); + File.Copy(archivePath, legacyArchivePath); + using var archive = ZipFile.Open(legacyArchivePath, ZipArchiveMode.Update); + var manifestEntry = archive.GetEntry("manifest.json") + ?? throw new InvalidOperationException("manifest.json entry was not found"); + JsonObject manifest; + using (var stream = manifestEntry.Open()) + { + manifest = JsonNode.Parse(stream)?.AsObject() + ?? throw new InvalidOperationException("manifest.json did not contain an object"); + } + manifest["scope"]?.AsObject().Remove("represents_entire_source_database"); + manifest.Remove("index_complete"); + manifest.Remove("index_incomplete_reasons"); + manifestEntry.Delete(); + var replacement = archive.CreateEntry("manifest.json", CompressionLevel.SmallestSize); + using var writer = new StreamWriter(replacement.Open(), new UTF8Encoding(false)); + writer.Write(manifest.ToJsonString()); + return legacyArchivePath; + } + + private static void SetArchiveTrustMetadata(string dbPath, int filesScanned) + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + writer.SetMeta(DbContext.IndexCompletenessMetaKey, "complete"); + writer.SetMeta(DbContext.IndexIncompleteReasonsMetaKey, string.Empty); + writer.SetMeta(DbContext.IndexedHeadCommitMetaKey, "source-full-head"); + writer.SetMeta(DbContext.IndexedHeadShaMetaKey, "source-head"); + writer.SetMeta(DbContext.CommitScopedFreshHeadShaMetaKey, "source-head"); + writer.SetMeta(DbContext.IndexedHeadBranchMetaKey, "main"); + writer.SetMeta(DbContext.IndexedHeadTimestampMetaKey, "2026-08-09T00:00:00Z"); + writer.SetMeta(DbContext.LastIndexRunModeMetaKey, "rebuild"); + writer.SetMeta(DbContext.LastIndexRunFilesScannedMetaKey, filesScanned.ToString(CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunPeakMemoryMbMetaKey, "1298"); + writer.SetMeta(DbContext.UnknownExtensionFileCountMetaKey, "2"); + writer.SetMeta( + DbContext.UnknownExtensionFilePathsMetaKey, + JsonSerializer.Serialize(new[] { "docs/archive.foo", "tools/cache.bar" })); + writer.SetMeta(DbContext.UnknownExtensionFilesTruncatedMetaKey, bool.FalseString); + writer.SetMeta( + DbContext.UnknownExtensionFilePathLimitMetaKey, + DbContext.UnknownExtensionFilePathSampleLimit.ToString(CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.UnknownExtensionExtensionCountsMetaKey, "{\".bar\":1,\".foo\":1}"); + writer.SetMeta(DbContext.UnknownExtensionCategoryCountsMetaKey, "{\"language_support\":2}"); + writer.SetMeta(DbContext.UnknownExtensionGroupsMetaKey, "[]"); + } + + private static void AssertPartialArchiveStatus(string dbPath) + { + var (exitCode, stdout, stderr) = ConsoleCapture.Capture(() => + ProgramRunner.Run(["status", "--db", dbPath, "--json"], appVersion: "test")); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + using var status = JsonDocument.Parse(stdout); + Assert.False(status.RootElement.GetProperty("index_complete").GetBoolean()); + Assert.Contains( + status.RootElement.GetProperty("index_incomplete_reasons").EnumerateArray(), + reason => reason.GetString() == ExportImportCommandRunner.PartialArchiveIncompleteReason); + Assert.False(status.RootElement.GetProperty("graph_data_current").GetBoolean()); + Assert.False(status.RootElement.GetProperty("reference_graph_complete").GetBoolean()); + Assert.True( + !status.RootElement.TryGetProperty("head_freshness", out var headFreshness) + || headFreshness.GetProperty("state").GetString() != "head_current"); + Assert.Equal(0, status.RootElement.GetProperty("unknown_extension_file_count").GetInt64()); + Assert.True( + !status.RootElement.TryGetProperty("last_index_run", out var lastIndexRun) + || lastIndexRun.ValueKind == JsonValueKind.Null); + Assert.Contains("INCOMPLETE", status.RootElement.GetProperty("summary").GetString(), StringComparison.Ordinal); + + var (checkExitCode, checkStdout, checkStderr) = ConsoleCapture.Capture(() => + ProgramRunner.Run(["status", "--check", "--db", dbPath, "--json"], appVersion: "test")); + Assert.NotEqual(CommandExitCodes.Success, checkExitCode); + Assert.Equal(string.Empty, checkStderr); + using var checkStatus = JsonDocument.Parse(checkStdout); + Assert.False(checkStatus.RootElement.GetProperty("index_complete").GetBoolean()); + Assert.Contains( + checkStatus.RootElement.GetProperty("index_incomplete_reasons").EnumerateArray(), + reason => reason.GetString() == ExportImportCommandRunner.PartialArchiveIncompleteReason); + Assert.True( + !checkStatus.RootElement.TryGetProperty("head_freshness", out var checkHeadFreshness) + || checkHeadFreshness.GetProperty("state").GetString() != "head_current"); + Assert.Contains("INCOMPLETE", checkStatus.RootElement.GetProperty("summary").GetString(), StringComparison.Ordinal); + } + private static void SetUnknownExtensionPathSamples(string dbPath, string[] paths) { using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); From f58c4bd2330da043e02f780caa46c9fbba64683c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 10 Aug 2026 02:33:52 +0900 Subject: [PATCH 2/4] Harden legacy archive trust import (#5053) --- .../ExportImportCommandRunner.ArchiveScope.cs | 53 ++++++------ .../Cli/ExportImportCommandRunner.Import.cs | 3 +- .../ExportImportCommandRunnerTests.cs | 80 +++++++++++++++++-- 3 files changed, 104 insertions(+), 32 deletions(-) diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs index 8004380b8..b33202b47 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs @@ -166,27 +166,43 @@ WHERE reference_id NOT IN (SELECT id FROM symbol_references) RepresentsEntireSourceDatabase: false); } - private static void ApplyImportedArchiveTrustMetadata( + private static ExportManifest ApplyImportedArchiveTrustMetadata( string databasePath, ExportManifest manifest, CancellationToken cancellationToken) { if (ManifestRepresentsEntireSourceDatabase(manifest)) - return; + return manifest; cancellationToken.ThrowIfCancellationRequested(); using var connection = new SqliteConnection(CreateUnpooledConnectionString(databasePath)); connection.Open(); - if (!ArchiveTrustMetadataRequiresNormalization(connection)) - return; + if (ArchiveTrustMetadataRequiresNormalization(connection)) + { + using var transaction = connection.BeginTransaction(); + ApplyPartialArchiveTrustMetadata(connection, transaction, cancellationToken); + transaction.Commit(); + } - using var transaction = connection.BeginTransaction(); - ApplyPartialArchiveTrustMetadata(connection, transaction, cancellationToken); - transaction.Commit(); + return NormalizeImportedArchiveTrustMetadata( + manifest, + ReadArchiveIncompleteReasons(connection)); } private static bool ArchiveTrustMetadataRequiresNormalization(SqliteConnection connection) { + using (var tableCommand = connection.CreateCommand()) + { + tableCommand.CommandText = """ + SELECT EXISTS( + SELECT 1 + FROM sqlite_master + WHERE type = 'table' AND name = 'codeindex_meta') + """; + if (Convert.ToInt64(tableCommand.ExecuteScalar(), CultureInfo.InvariantCulture) == 0) + return true; + } + if (!string.Equals( ReadMetaString(connection, DbContext.IndexCompletenessMetaKey), "incomplete", @@ -248,27 +264,13 @@ private static bool HasPartialArchiveIncompleteReason(string? rawReasons) } } - private static ExportManifest NormalizeImportedArchiveTrustMetadata(ExportManifest manifest) + private static ExportManifest NormalizeImportedArchiveTrustMetadata( + ExportManifest manifest, + string[]? persistedIncompleteReasons) { if (ManifestRepresentsEntireSourceDatabase(manifest)) return manifest; - var reasons = new List(MaxArchiveIncompleteReasons); - var totalChars = 0; - foreach (var reason in manifest.IndexIncompleteReasons ?? []) - { - if (string.Equals(reason, PartialArchiveIncompleteReason, StringComparison.Ordinal)) - continue; - if (reasons.Count >= MaxArchiveIncompleteReasons - 1 - || totalChars + reason.Length + PartialArchiveIncompleteReason.Length - > MaxArchiveIncompleteReasonsTotalChars) - { - break; - } - reasons.Add(reason); - totalChars += reason.Length; - } - reasons.Add(PartialArchiveIncompleteReason); return manifest with { IndexedHeadSha = null, @@ -280,7 +282,8 @@ private static ExportManifest NormalizeImportedArchiveTrustMetadata(ExportManife UnknownExtensionFileSampleCount = 0, UnknownExtensionFileSampleTruncated = false, IndexComplete = false, - IndexIncompleteReasons = reasons.ToArray(), + IndexIncompleteReasons = persistedIncompleteReasons + ?? [PartialArchiveIncompleteReason], }; } diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.Import.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.Import.cs index 9ce82f49e..9dfca313d 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.Import.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.Import.cs @@ -115,11 +115,10 @@ public static int RunImport(string[] args, JsonSerializerOptions jsonOptions, Ca AddImportValidationPhase(validationPhases, PhaseSqliteValidate); SqliteConnection.ClearAllPools(); - ApplyImportedArchiveTrustMetadata( + importedManifest = ApplyImportedArchiveTrustMetadata( tempPath, importedManifest ?? throw new InvalidDataException("archive manifest was not loaded"), cancellationToken); - importedManifest = NormalizeImportedArchiveTrustMetadata(importedManifest); SqliteConnection.ClearAllPools(); if (prunePaths) diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index e8578fb65..5f89356d3 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -1117,7 +1117,16 @@ public void RunImport_FullAndLegacyArchivesMaterializeConservativeTrust_Issue505 Assert.Equal("1", ReadMetaValue(fullImportDbPath, DbContext.LastIndexRunFilesScannedMetaKey)); Assert.Equal("2", ReadMetaValue(fullImportDbPath, DbContext.UnknownExtensionFileCountMetaKey)); - var legacyArchivePath = CopyArchiveWithoutTrustScopeMarker(projectRoot, pristineArchivePath); + using (var sourceDb = new DbContext(DbOpenIntent.WriteIndex, sourceDbPath)) + { + var writer = new DbWriter(sourceDb.Connection); + writer.SetMeta(DbContext.IndexCompletenessMetaKey, "incomplete"); + writer.SetMeta( + DbContext.IndexIncompleteReasonsMetaKey, + "[\"legacy_source_incomplete\"]"); + } + var incompleteArchivePath = ExportArchive(projectRoot, sourceDbPath); + var legacyArchivePath = CopyArchiveWithoutTrustScopeMarker(projectRoot, incompleteArchivePath); var legacyImportDbPath = Path.Combine(projectRoot, "legacy", "codeindex.db"); var (legacyExitCode, legacyStdout, legacyStderr) = ConsoleCapture.Capture(() => ExportImportCommandRunner.RunImport( @@ -1129,9 +1138,12 @@ public void RunImport_FullAndLegacyArchivesMaterializeConservativeTrust_Issue505 using (var legacyResult = JsonDocument.Parse(legacyStdout)) { Assert.False(legacyResult.RootElement.GetProperty("index_complete").GetBoolean()); - Assert.Contains( - legacyResult.RootElement.GetProperty("index_incomplete_reasons").EnumerateArray(), - reason => reason.GetString() == ExportImportCommandRunner.PartialArchiveIncompleteReason); + Assert.Equal( + ["legacy_source_incomplete", ExportImportCommandRunner.PartialArchiveIncompleteReason], + legacyResult.RootElement + .GetProperty("index_incomplete_reasons") + .EnumerateArray() + .Select(reason => reason.GetString())); var legacyScope = legacyResult.RootElement.GetProperty("scope"); Assert.True( !legacyScope.TryGetProperty("represents_entire_source_database", out var wholeSource) @@ -1143,7 +1155,37 @@ public void RunImport_FullAndLegacyArchivesMaterializeConservativeTrust_Issue505 Assert.Null(ReadMetaValue(legacyImportDbPath, DbContext.CommitScopedFreshHeadShaMetaKey)); Assert.Null(ReadMetaValue(legacyImportDbPath, DbContext.LastIndexRunFilesScannedMetaKey)); Assert.Equal("0", ReadMetaValue(legacyImportDbPath, DbContext.UnknownExtensionFileCountMetaKey)); + Assert.Equal( + "[\"legacy_source_incomplete\",\"partial_archive\"]", + ReadMetaValue(legacyImportDbPath, DbContext.IndexIncompleteReasonsMetaKey)); AssertPartialArchiveStatus(legacyImportDbPath); + + var noMetaArchivePath = CopyArchiveWithoutTrustScopeMarker( + projectRoot, + pristineArchivePath, + removeMetadataTable: true); + var noMetaImportDbPath = Path.Combine(projectRoot, "no-meta", "codeindex.db"); + var (noMetaExitCode, noMetaStdout, noMetaStderr) = ConsoleCapture.Capture(() => + ExportImportCommandRunner.RunImport( + [noMetaArchivePath, "--db", noMetaImportDbPath, "--json"], + jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, noMetaExitCode); + Assert.Equal(string.Empty, noMetaStderr); + using (var noMetaResult = JsonDocument.Parse(noMetaStdout)) + { + Assert.False(noMetaResult.RootElement.GetProperty("index_complete").GetBoolean()); + Assert.Equal( + [ExportImportCommandRunner.PartialArchiveIncompleteReason], + noMetaResult.RootElement + .GetProperty("index_incomplete_reasons") + .EnumerateArray() + .Select(reason => reason.GetString())); + } + Assert.Equal( + "[\"partial_archive\"]", + ReadMetaValue(noMetaImportDbPath, DbContext.IndexIncompleteReasonsMetaKey)); + AssertPartialArchiveStatus(noMetaImportDbPath); } finally { @@ -2271,7 +2313,10 @@ private static string ExportArchive(string projectRoot, string dbPath) return archivePath; } - private static string CopyArchiveWithoutTrustScopeMarker(string projectRoot, string archivePath) + private static string CopyArchiveWithoutTrustScopeMarker( + string projectRoot, + string archivePath, + bool removeMetadataTable = false) { var legacyArchivePath = Path.Combine(projectRoot, $"legacy-{Guid.NewGuid():N}.cdidx.zip"); File.Copy(archivePath, legacyArchivePath); @@ -2287,6 +2332,31 @@ private static string CopyArchiveWithoutTrustScopeMarker(string projectRoot, str manifest["scope"]?.AsObject().Remove("represents_entire_source_database"); manifest.Remove("index_complete"); manifest.Remove("index_incomplete_reasons"); + if (removeMetadataTable) + { + var databaseEntry = archive.GetEntry("codeindex.db") + ?? throw new InvalidOperationException("codeindex.db entry was not found"); + var stagedDatabasePath = Path.Combine(projectRoot, $"legacy-{Guid.NewGuid():N}.db"); + databaseEntry.ExtractToFile(stagedDatabasePath); + databaseEntry.Delete(); + using (var connection = new SqliteConnection( + new SqliteConnectionStringBuilder { DataSource = stagedDatabasePath }.ConnectionString)) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "DROP TABLE codeindex_meta"; + command.ExecuteNonQuery(); + } + SqliteConnection.ClearAllPools(); + manifest["database_sha256"] = Convert.ToHexString( + SHA256.HashData(File.ReadAllBytes(stagedDatabasePath))) + .ToLowerInvariant(); + var replacementDatabase = archive.CreateEntry("codeindex.db", CompressionLevel.SmallestSize); + using (var source = File.OpenRead(stagedDatabasePath)) + using (var destination = replacementDatabase.Open()) + source.CopyTo(destination); + File.Delete(stagedDatabasePath); + } manifestEntry.Delete(); var replacement = archive.CreateEntry("manifest.json", CompressionLevel.SmallestSize); using var writer = new StreamWriter(replacement.Open(), new UTF8Encoding(false)); From 48638743ae3c7a0ad1880476f5381a40df82e7c6 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 10 Aug 2026 02:54:57 +0900 Subject: [PATCH 3/4] Preserve partial archive trust on scoped updates (#5053) --- DEVELOPER_GUIDE.md | 4 +- TESTING_GUIDE.md | 4 +- USER_GUIDE.md | 16 +++-- changelog.d/unreleased/5053.fixed.md | 6 +- .../ExportImportCommandRunner.ArchiveScope.cs | 69 +++++++------------ .../Cli/IndexCommandRunner.Update.cs | 23 +++---- src/CodeIndex/Cli/IndexCommandRunner.cs | 8 ++- .../ExportImportCommandRunnerTests.cs | 57 +++++++++++++-- 8 files changed, 107 insertions(+), 80 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 8aa2c36de..a9a02584f 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -56,7 +56,7 @@ Development contracts: | Binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | Database compatibility across `cdidx` binary upgrades and downgrades is documented there. Keep that policy updated whenever readiness bits, `codeindex_meta` contract stamps, or rebuild requirements change. | | Fold backfill preview and recovery | `backfill-fold --dry-run`; `backfill-fold --checkpoint`; MCP `backfill_fold` with `dry_run: true` or `force: true` | Dry-run previews folded-key rows without mutating the DB or stamping FoldReady. `--checkpoint` explicitly preserves a snapshot even when CLI preflight finds no mutation, while the default completed no-op leaves no checkpoint artifacts. MCP accepts the same preview and can force rewriting all folded keys when an operator needs to recover from suspicious fold metadata or row state even though the stored version/fingerprint appears current. Non-dry-run row rewrites are resumable after interruption: completed row updates remain durable, and final FoldReady metadata is stamped only after verification succeeds. MCP responses include `progress.rows_done`, `progress.rows_total`, and `progress.fraction` so clients can report and retry long backfills. | -Portable archive trust is scope-aware. Unfiltered exports set `scope.represents_entire_source_database` and preserve completeness, indexed-HEAD, run, and unknown-extension metadata. Filtered exports normalize only the archive snapshot to `index_complete=false` with `partial_archive`, remove source-wide HEAD/run provenance, and reset unknown-extension summaries. Import repeats that normalization for legacy manifests without scope metadata, while current full-snapshot manifests preserve trust. +Portable archive trust is scope-aware. Unfiltered exports set `scope.represents_entire_source_database` and preserve completeness, indexed-HEAD, run, and unknown-extension metadata. Filtered exports normalize only the archive snapshot to `index_complete=false` with `partial_archive`, remove source-wide HEAD/run provenance, and omit unavailable unknown-extension summaries. Import repeats that normalization for legacy manifests without scope metadata, while current full-snapshot manifests preserve trust. Scoped indexing of a partial archive falls back to a full workspace scan before it may clear `partial_archive`. Checkpoint plan drift detection covers DB/WAL/SHM content changes and sidecar appearance or disappearance through the final pre-publication validation. The plan's `uncertainty` value records the remaining post-validation race; copied outputs are independently hash-verified against the plan before atomic publication. DB/WAL/SHM candidates must pass native regular-file type validation before hashing so Unix FIFOs cannot block planning. `metadata_policy` reports `owner_only_files_and_directories` on POSIX and the actually inherited `inherited_windows_acls` policy on Windows. A database payload whose output name collides with `manifest.txt`, including filesystem-equivalent casing, makes the plan not ready and is rejected before mutation. @@ -3719,7 +3719,7 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま | binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | `cdidx` binary の upgrade / downgrade をまたぐ database compatibility を記載します。readiness bit、`codeindex_meta` contract stamp、rebuild requirement を変える場合は、この policy も更新してください。 | | Fold backfill の preview / recovery | `backfill-fold --dry-run`; `backfill-fold --checkpoint`; MCP `backfill_fold` の `dry_run: true` または `force: true` | dry-run は DB を変更せず FoldReady stamp も書かずに、rewrite 対象の folded-key row をプレビューします。CLI preflight でmutation不要と判断された場合でもsnapshotを明示的に保存するには `--checkpoint` を使います。既定の完了済みno-opはcheckpoint artifactを作りません。MCP も同じ preview を受け付け、stored version / fingerprint が current に見える場合でも suspicious な fold metadata や row state を復旧するため `force: true` を受け付けます。non-dry-run rewrite は中断後に resume でき、完了済み row update は durable に残り、最終 FoldReady metadata は verification 成功後にだけ stamp されます。MCP response は `progress.rows_done`、`progress.rows_total`、`progress.fraction` を含みます。 | -portable archive の trust は scope を考慮します。filter なし export は `scope.represents_entire_source_database` を設定し、completeness、indexed-HEAD、run、unknown-extension metadata を維持します。filter 済み export は archive snapshot だけを `index_complete=false` / `partial_archive` に正規化し、source 全体に対する HEAD / run provenance を削除して unknown-extension summary を reset します。scope metadata がない legacy manifest は import 時に同じ正規化を行い、現行の full-snapshot manifest は trust を維持します。 +portable archive の trust は scope を考慮します。filter なし export は `scope.represents_entire_source_database` を設定し、completeness、indexed-HEAD、run、unknown-extension metadata を維持します。filter 済み export は archive snapshot だけを `index_complete=false` / `partial_archive` に正規化し、source 全体に対する HEAD / run provenance を削除して、未計測の unknown-extension summary を省略します。scope metadata がない legacy manifest は import 時に同じ正規化を行い、現行の full-snapshot manifest は trust を維持します。partial archive に対する scoped index は `partial_archive` を解除する前に full workspace scan へ fallback します。 checkpoint plan の drift 検出は、publish 前の最終検証までに起きる DB/WAL/SHM content の変更と sidecar の出現・消失を対象にします。plan の `uncertainty` 値は最終検証後に残る race を記録し、copy 済み output は atomic publish 前に plan の hash と個別に照合されます。DB/WAL/SHM 候補は hash 読み取り前に native regular-file type validation を通すため、Unix FIFO が plan 作成を停止させることはありません。`metadata_policy` は POSIX では `owner_only_files_and_directories`、Windows では実際に継承される `inherited_windows_acls` policy を報告します。database payload の output 名が filesystem 上で同一の大小文字を含め `manifest.txt` と衝突する場合、plan は not ready となり、変更前に拒否されます。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 6f70d3b37..12db62d2d 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -225,7 +225,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Inspect graph-section coverage must compare name and path/line resolution through the same persisted candidate ID, keep ambiguous overload and partial-family bundles isolated, assert independent total/returned/truncated metadata for references, callers, and callees (including empty sections), and replay a query-, page-size-, and generation-bound cursor across the smallest two-row page boundary. Put inbound callers in another file to prove the location path is only a locator, seed equal-rank same-line callees to pin the complete identity tie-breakers, reject page-size changes before candidate lookup, reject inspect cursors in another command, and verify the same envelopes in MCP `analyze_symbol`. Quiet-flag coverage in `ProgramCliTests.cs` reuses one seeded symbols database across text, NDJSON, and JSON-array modes and compares stdout with and without a trailing quiet alias, proving that quiet mode changes only informational stderr. Archive-import validation coverage reuses one pristine database export across read-only dry-run and check modes with distinct destinations. Rejection coverage copies one pristine export into manifest-count, database-hash, and user-version variants before mutating any ZIP so one corruption cannot contaminate another. - Archive success-path coverage seeds one metadata-rich database and shares its pristine export across scoped manifest inspection, a default import into a nonexistent destination, a copied legacy-manifest import, and a separate `--no-backup` replacement. Scope coverage must prove a filtered snapshot is incomplete with `partial_archive`, has no source-wide HEAD/run or unknown-extension trust, fails `status --check`, and leaves the source DB unchanged. Full-import coverage preserves those trust fields, while the copied scope-less manifest materializes the same conservative partial state. Keep the default import and replacement as distinct CLI calls and destinations, and never mutate the pristine archive. + Archive success-path coverage seeds one metadata-rich database and shares its pristine export across scoped manifest inspection, a default import into a nonexistent destination, a copied legacy-manifest import, and a separate `--no-backup` replacement. Scope coverage must prove a filtered snapshot is incomplete with `partial_archive`, has no source-wide HEAD/run trust, omits unavailable unknown-extension coverage, fails `status --check`, and leaves the source DB unchanged. A later scoped index must fall back to a full workspace scan before restoring complete trust. Full-import coverage preserves those trust fields, while the copied scope-less manifest materializes the same conservative partial state. Keep the default import and replacement as distinct CLI calls and destinations, and never mutate the pristine archive. Doctor full-inventory coverage keeps composed filter selection, filtered summary counts, exact UTF-8 byte-budget boundaries, and structured overflow errors together in `ProgramRunnerTests`; license JSON remains a subprocess contract in `ProgramCliTests` so immediate-command dispatch and the published field names are both exercised. Ctags export JSON coverage reuses one seeded database for default and `--include-generated` variants, asserts the fixed skip-reason keys sum to `skipped_count`, and keeps the missing-`files.generated` degradation in a separate legacy-schema fixture. Dry-run JSON coverage for ambiguous `.h` files locks the bounded `language_detections` entries and their stable source/confidence codes without mutating the index. @@ -1223,7 +1223,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" inspect graph-section coverage では、name と path/line resolution が同じ persisted candidate ID を通ること、曖昧な overload と partial-family bundle が分離されること、references / callers / callees の独立した total / returned / truncated metadata（空 section を含む）を検証してください。inbound caller は別ファイルに置いて location path が locator にすぎないことを証明し、最小の2行 page 境界で query / page size / generation に束縛された cursor を再利用します。同順位かつ同じ行の callee で完全な identity tie-breaker を固定し、candidate lookup より前に page-size 変更を拒否し、別 command では inspect cursor を拒否してください。MCP `analyze_symbol` でも同じ envelope を確認します。 `ProgramCliTests.cs` の quiet flag coverage は1つの seeded symbols database を text、NDJSON、JSON array の各 mode で再利用し、末尾に quiet alias を付けた場合と付けない場合の stdout を比較して、quiet mode が informational stderr だけを変えることを固定します。 archive import validation coverage は、read-only な dry-run / check mode で1つの pristine database export を別々の destination から共有してください。拒否 coverage では、mutation 前に1つの pristine export を manifest-count、database-hash、user-version 用の3つの ZIP へコピーし、ある corruption が別 case を汚染しないようにします。 - archive success-path coverage は、metadata-rich な database を1回 seed し、その pristine export を scoped manifest inspection、存在しない destination への default import、コピーした legacy manifest の import、別 destination への `--no-backup` replacement で共有します。scope coverage では filter 済み snapshot が `partial_archive` により incomplete となり、source 全体に対する HEAD / run / unknown-extension trust を持たず、`status --check` が失敗し、source DB が変更されないことを検証してください。full import はそれらの trust field を維持し、scope を除いた manifest のコピーは同じ保守的な partial state を materialize します。default import と replacement は別々の CLI 呼び出しと destination に保ち、pristine archive を直接変更しないでください。 + archive success-path coverage は、metadata-rich な database を1回 seed し、その pristine export を scoped manifest inspection、存在しない destination への default import、コピーした legacy manifest の import、別 destination への `--no-backup` replacement で共有します。scope coverage では filter 済み snapshot が `partial_archive` により incomplete となり、source 全体に対する HEAD / run trust を持たず、未計測の unknown-extension coverage を省略し、`status --check` が失敗し、source DB が変更されないことを検証してください。後続の scoped index は complete trust を復元する前に full workspace scan へ fallback しなければなりません。full import はそれらの trust field を維持し、scope を除いた manifest のコピーは同じ保守的な partial state を materialize します。default import と replacement は別々の CLI 呼び出しと destination に保ち、pristine archive を直接変更しないでください。 doctor full-inventory coverage では、合成 filter の選択、filtered summary 件数、UTF-8 byte budget の exact boundary、structured overflow error を `ProgramRunnerTests` にまとめます。license JSON は `ProgramCliTests` の subprocess contract として、immediate-command dispatch と公開 field 名を同時に検証します。 ctags export JSON coverage は1つの seeded database を既定と `--include-generated` variant で再利用し、固定された skip-reason key の合計が `skipped_count` と一致することを検証します。`files.generated` がない場合の縮退は別の legacy-schema fixture に保ってください。 曖昧な `.h` に対する dry-run JSON coverage は、index を変更せず、上限付き `language_detections` entry と安定した判定元・信頼度 code を固定します。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index e6c0402d2..15ab4265f 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -511,9 +511,11 @@ full archive and preserves the source database's index completeness, indexed-HEAD provenance, run telemetry, and unknown-extension summary. A filtered archive is instead stamped `index_complete: false` with the stable `partial_archive` reason, clears source-wide indexed-HEAD and run metadata, and -resets unknown-extension summaries so `status` cannot present the subset as a -fresh full index. This normalization applies only to the exported snapshot and -does not mutate the source database. +omits unavailable unknown-extension summaries so `status` cannot present the +subset as a fresh full index or an authoritative zero-result scan. This +normalization applies only to the exported snapshot and does not mutate the +source database. A later scoped index request falls back to a full workspace +scan before clearing `partial_archive`. Portable export refuses an existing destination by default; pass `--overwrite` only when replacing it is intentional. The archive is built in an owner-only sibling temporary file and atomically published, and POSIX archives are verified @@ -4014,9 +4016,11 @@ file count が含まれます。scope には `represents_entire_source_database` scope flag を指定しない full archive は source database の index completeness、 indexed-HEAD provenance、run telemetry、unknown-extension summary を維持します。一方、 filter 済み archive は `index_complete: false` と stable reason `partial_archive` を記録し、 -source 全体に対する indexed-HEAD / run metadata を消去して unknown-extension summary を -reset するため、`status` が subset を fresh な full index として表示することはありません。 -この正規化は export snapshot だけに適用され、source database は変更しません。 +source 全体に対する indexed-HEAD / run metadata を消去し、未計測の unknown-extension +summary を省略するため、`status` が subset を fresh な full index や authoritative な +0 件 scan として表示することはありません。この正規化は export snapshot だけに適用され、 +source database は変更しません。後続の scoped index request は `partial_archive` を解除する前に +full workspace scan へ fallback します。 portable export は既存 destination を既定で拒否します。意図して置き換える場合だけ `--overwrite` を指定してください。archive は owner-only の sibling temporary file に 構築して atomic に publish し、POSIX では mode `0600` であることも検証します。 diff --git a/changelog.d/unreleased/5053.fixed.md b/changelog.d/unreleased/5053.fixed.md index c40215228..9fb280373 100644 --- a/changelog.d/unreleased/5053.fixed.md +++ b/changelog.d/unreleased/5053.fixed.md @@ -6,6 +6,8 @@ affected: - src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs - src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs - src/CodeIndex/Cli/ExportImportManifestCodec.cs + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs - tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs - USER_GUIDE.md - DEVELOPER_GUIDE.md @@ -14,8 +16,8 @@ affected: ## English -- **Filtered portable archives now report conservative trust metadata (#5053)** — Scoped exports and legacy scope-less imports are marked incomplete with `partial_archive`, clear source-wide HEAD/run provenance, and reset unknown-extension summaries, while full unfiltered archives preserve their trust metadata. +- **Filtered portable archives now report conservative trust metadata (#5053)** — Scoped exports and legacy scope-less imports are marked incomplete with `partial_archive`, clear source-wide HEAD/run provenance, omit unavailable unknown-extension summaries, and require a full workspace scan before scoped indexing can restore complete trust, while full unfiltered archives preserve their trust metadata. ## 日本語 -- **filter 済み portable archive が保守的な trust metadata を報告するようになりました (#5053)** — scope 付き export と scope 情報がない legacy import は `partial_archive` により incomplete とし、source 全体に対する HEAD / run provenance を削除して unknown-extension summary を reset します。filter なしの full archive は従来の trust metadata を維持します。 +- **filter 済み portable archive が保守的な trust metadata を報告するようになりました (#5053)** — scope 付き export と scope 情報がない legacy import は `partial_archive` により incomplete とし、source 全体に対する HEAD / run provenance と未計測の unknown-extension summary を削除します。scoped index が complete trust を復元する前には full workspace scan を必須とし、filter なしの full archive は従来の trust metadata を維持します。 diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs index b33202b47..24be49163 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs @@ -208,16 +208,7 @@ FROM sqlite_master "incomplete", StringComparison.Ordinal) || !HasPartialArchiveIncompleteReason( - ReadMetaString(connection, DbContext.IndexIncompleteReasonsMetaKey)) - || ReadMetaString(connection, DbContext.UnknownExtensionFileCountMetaKey) != "0" - || ReadMetaString(connection, DbContext.UnknownExtensionFilePathsMetaKey) != "[]" - || !bool.TryParse( - ReadMetaString(connection, DbContext.UnknownExtensionFilesTruncatedMetaKey), - out var unknownFilesTruncated) - || unknownFilesTruncated - || ReadMetaString(connection, DbContext.UnknownExtensionExtensionCountsMetaKey) != "{}" - || ReadMetaString(connection, DbContext.UnknownExtensionCategoryCountsMetaKey) != "{}" - || ReadMetaString(connection, DbContext.UnknownExtensionGroupsMetaKey) != "[]") + ReadMetaString(connection, DbContext.IndexIncompleteReasonsMetaKey))) { return true; } @@ -232,7 +223,14 @@ OR key GLOB 'last_failed_index_run_*' OR key IN ( 'commit_scoped_fresh_head_sha', 'last_full_scan_elapsed_ms', - 'last_workspace_freshened_at') + 'last_workspace_freshened_at', + 'unknown_extension_file_count', + 'unknown_extension_file_paths_json', + 'unknown_extension_files_truncated', + 'unknown_extension_file_path_limit', + 'unknown_extension_extension_counts_json', + 'unknown_extension_category_counts_json', + 'unknown_extension_groups_json') """; return Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture) != 0; } @@ -276,11 +274,13 @@ private static ExportManifest NormalizeImportedArchiveTrustMetadata( IndexedHeadSha = null, IndexedHeadBranch = null, IndexedHeadTimestamp = null, - UnknownExtensionFileCount = 0, + UnknownExtensionFileCount = null, UnknownExtensionFiles = null, - UnknownExtensionFilesTruncated = false, - UnknownExtensionFileSampleCount = 0, - UnknownExtensionFileSampleTruncated = false, + UnknownExtensionFilesTruncated = null, + UnknownExtensionFilePathLimit = null, + UnknownExtensionFileSampleCount = null, + UnknownExtensionFileSampleLimit = null, + UnknownExtensionFileSampleTruncated = null, IndexComplete = false, IndexIncompleteReasons = persistedIncompleteReasons ?? [PartialArchiveIncompleteReason], @@ -318,7 +318,14 @@ OR key GLOB 'last_failed_index_run_*' OR key IN ( 'commit_scoped_fresh_head_sha', 'last_full_scan_elapsed_ms', - 'last_workspace_freshened_at'); + 'last_workspace_freshened_at', + 'unknown_extension_file_count', + 'unknown_extension_file_paths_json', + 'unknown_extension_files_truncated', + 'unknown_extension_file_path_limit', + 'unknown_extension_extension_counts_json', + 'unknown_extension_category_counts_json', + 'unknown_extension_groups_json'); INSERT INTO codeindex_meta(key, value) VALUES (@indexCompletenessKey, 'incomplete') @@ -327,40 +334,10 @@ INSERT INTO codeindex_meta(key, value) INSERT INTO codeindex_meta(key, value) VALUES (@indexIncompleteReasonsKey, @indexIncompleteReasons) ON CONFLICT(key) DO UPDATE SET value = excluded.value; - - INSERT INTO codeindex_meta(key, value) - VALUES (@unknownExtensionFileCountKey, '0') - ON CONFLICT(key) DO UPDATE SET value = excluded.value; - - INSERT INTO codeindex_meta(key, value) - VALUES (@unknownExtensionFilePathsKey, '[]') - ON CONFLICT(key) DO UPDATE SET value = excluded.value; - - INSERT INTO codeindex_meta(key, value) - VALUES (@unknownExtensionFilesTruncatedKey, 'False') - ON CONFLICT(key) DO UPDATE SET value = excluded.value; - - INSERT INTO codeindex_meta(key, value) - VALUES (@unknownExtensionExtensionCountsKey, '{}') - ON CONFLICT(key) DO UPDATE SET value = excluded.value; - - INSERT INTO codeindex_meta(key, value) - VALUES (@unknownExtensionCategoryCountsKey, '{}') - ON CONFLICT(key) DO UPDATE SET value = excluded.value; - - INSERT INTO codeindex_meta(key, value) - VALUES (@unknownExtensionGroupsKey, '[]') - ON CONFLICT(key) DO UPDATE SET value = excluded.value; """; SqliteCommandPolicy.Add(command, "@indexCompletenessKey", DbContext.IndexCompletenessMetaKey); SqliteCommandPolicy.Add(command, "@indexIncompleteReasonsKey", DbContext.IndexIncompleteReasonsMetaKey); SqliteCommandPolicy.Add(command, "@indexIncompleteReasons", incompleteReasonsJson); - SqliteCommandPolicy.Add(command, "@unknownExtensionFileCountKey", DbContext.UnknownExtensionFileCountMetaKey); - SqliteCommandPolicy.Add(command, "@unknownExtensionFilePathsKey", DbContext.UnknownExtensionFilePathsMetaKey); - SqliteCommandPolicy.Add(command, "@unknownExtensionFilesTruncatedKey", DbContext.UnknownExtensionFilesTruncatedMetaKey); - SqliteCommandPolicy.Add(command, "@unknownExtensionExtensionCountsKey", DbContext.UnknownExtensionExtensionCountsMetaKey); - SqliteCommandPolicy.Add(command, "@unknownExtensionCategoryCountsKey", DbContext.UnknownExtensionCategoryCountsMetaKey); - SqliteCommandPolicy.Add(command, "@unknownExtensionGroupsKey", DbContext.UnknownExtensionGroupsMetaKey); command.ExecuteNonQuery(); cancellationToken.ThrowIfCancellationRequested(); } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 99eca206c..ea5254372 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -33,7 +33,7 @@ private static int RunUpdateMode( JsonSerializerOptions jsonOptions, int priorReadiness, bool priorIndexComplete, - bool priorFileIndexIncomplete, + bool priorScopedUpdateRequiresFullScan, bool priorSymbolsOnlyGraphOmitted, string? priorFoldVersion, string? priorFoldFingerprint, @@ -105,7 +105,7 @@ private static int RunUpdateMode( var typeScriptJavaScriptConfigChanged = ContainsJavaScriptTypeScriptConfigPath(targetPaths); var extractorConfigurationChanged = ContainsExtractorConfigurationPath(projectRoot, targetPaths); var ambiguousLanguageProjectMarkerChanged = targetPaths.Any(FileIndexer.IsAmbiguousLanguageProjectMarkerPath); - if (priorFileIndexIncomplete + if (priorScopedUpdateRequiresFullScan || relevantIgnoreFileChanged || ContainsIgnoreFilePath(targetPaths) || typeScriptJavaScriptConfigChanged @@ -117,8 +117,8 @@ private static int RunUpdateMode( if (!options.Json && !options.Quiet) { - var reason = priorFileIndexIncomplete - ? "an earlier partial index still has unresolved file failures" + var reason = priorScopedUpdateRequiresFullScan + ? "an earlier partial index still requires a full workspace scan" : extractorConfigurationChanged ? "extractor configuration changes" : typeScriptJavaScriptConfigChanged @@ -130,14 +130,13 @@ private static int RunUpdateMode( CommandOutputWriter.WriteLine(); } - // A scoped pass cannot prove that failures outside its target set recovered, and - // the partial pass deliberately cleared workspace-wide readiness stamps. Reuse the - // normal incremental full-scan path until every failed file has been revisited; this - // preserves the failure on unrelated updates and restores all contracts without a - // destructive rebuild once the source problem is fixed. Issue #4609 review. - // scoped pass だけでは対象外の失敗回復を証明できず、partial pass は workspace 全体の - // readiness を落としている。失敗解消までは通常の incremental full-scan に切り替え、 - // 無関係 update で failure を消さず、修正後は rebuild なしで全 contract を復元する。 + // A scoped pass cannot prove that failures or archive omissions outside its target + // set recovered. Reuse the normal incremental full-scan path until every file has + // been revisited; this preserves partial trust on unrelated updates and restores all + // contracts without a destructive rebuild. Issues #4609 and #5053. + // scoped pass だけでは対象外の failure / archive omission の回復を証明できない。 + // 全 file を再確認するまでは通常の incremental full-scan に切り替え、無関係 update で + // partial trust を消さず、destructive rebuild なしで全 contract を復元する。 return RunFullScan( db, writer, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 2d3e1532b..7c3512a38 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -446,11 +446,13 @@ private static int RunInitialIndex(IndexRunExecutionContext context) PriorMeta(DbContext.IndexCompletenessMetaKey), "complete", StringComparison.OrdinalIgnoreCase); - var priorFileIndexIncomplete = string.Equals( + var priorScopedUpdateRequiresFullScan = string.Equals( PriorMeta(DbContext.IndexCompletenessMetaKey), "incomplete", StringComparison.OrdinalIgnoreCase) - && priorIndexIncompleteReasons?.Contains("file_index_error", StringComparer.Ordinal) == true; + && priorIndexIncompleteReasons?.Any(reason => + reason is "file_index_error" + or ExportImportCommandRunner.PartialArchiveIncompleteReason) == true; var priorHotspotFamilyVersions = GetHotspotFamilyMetaSnapshot(db, DbContext.GetHotspotFamilyVersionMetaKey); var priorHotspotFamilyMarkerFingerprints = GetHotspotFamilyMetaSnapshot(db, DbContext.GetHotspotFamilyMarkerFingerprintMetaKey); var priorIndexedProjectRoot = PriorMeta(DbContext.IndexedProjectRootMetaKey); @@ -491,7 +493,7 @@ private static int RunInitialIndex(IndexRunExecutionContext context) var projectRoot = Path.GetFullPath(options.ProjectPath!); initialExitCode = isUpdateMode - ? RunUpdateMode(db, writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorIndexComplete, priorFileIndexIncomplete, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHdlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints!, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, indexCancellation.Token) + ? RunUpdateMode(db, writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorIndexComplete, priorScopedUpdateRequiresFullScan, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHdlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints!, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, indexCancellation.Token) : RunFullScan(db, writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorIndexComplete, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHdlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, showNextSteps: !databaseExistedBeforeIndex, indexCancellation.Token); if (initialExitCode == CommandExitCodes.Success) { diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index 5f89356d3..4dd40728c 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -968,12 +968,17 @@ public void RunExportArchive_DefaultRefusesDanglingSymlinkDestination_Issue4827( } [Fact] - public void RunExportArchive_AppliesProjectPathLanguageAndTestScope_Issue4714() + public void RunExportArchive_AppliesProjectPathLanguageAndTestScope_Issue4714_Issue5053() { var projectRoot = TestProjectHelper.CreateTempProject("export_archive_scope"); try { TestProjectHelper.WriteTextFile(projectRoot, "src/App/App.csproj", ""); + TestProjectHelper.WriteTextFile(projectRoot, "src/App/App.cs", "public class AppType { }"); + TestProjectHelper.WriteTextFile(projectRoot, "src/shared/Shared.cs", "public class SharedType { }"); + TestProjectHelper.WriteTextFile(projectRoot, "src/Other/Other.cs", "public class OtherType { }"); + TestProjectHelper.WriteTextFile(projectRoot, "src/App/tool.py", "def tool(): pass"); + TestProjectHelper.WriteTextFile(projectRoot, "src/App/tests/AppTests.cs", "public class AppTests { }"); var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); TestProjectHelper.InsertIndexedFile(dbPath, "src/App/App.cs", "csharp", "public class AppType { }"); TestProjectHelper.InsertIndexedFile(dbPath, "src/shared/Shared.cs", "csharp", "public class SharedType { }"); @@ -1012,7 +1017,7 @@ public void RunExportArchive_AppliesProjectPathLanguageAndTestScope_Issue4714() Assert.Contains( resultManifest.GetProperty("index_incomplete_reasons").EnumerateArray(), reason => reason.GetString() == ExportImportCommandRunner.PartialArchiveIncompleteReason); - Assert.Equal(0, resultManifest.GetProperty("unknown_extension_file_count").GetInt64()); + Assert.Equal(JsonValueKind.Null, resultManifest.GetProperty("unknown_extension_file_count").ValueKind); Assert.Equal(string.Empty, ReadMetaValue(dbPath, DbContext.IndexIncompleteReasonsMetaKey)); Assert.Equal("source-head", ReadMetaValue(dbPath, DbContext.IndexedHeadShaMetaKey)); Assert.Equal("source-head", ReadMetaValue(dbPath, DbContext.CommitScopedFreshHeadShaMetaKey)); @@ -1051,8 +1056,8 @@ public void RunExportArchive_AppliesProjectPathLanguageAndTestScope_Issue4714() Assert.Null(ReadMetaValue(extractedDb, DbContext.IndexedHeadShaMetaKey)); Assert.Null(ReadMetaValue(extractedDb, DbContext.CommitScopedFreshHeadShaMetaKey)); Assert.Null(ReadMetaValue(extractedDb, DbContext.LastIndexRunFilesScannedMetaKey)); - Assert.Equal("0", ReadMetaValue(extractedDb, DbContext.UnknownExtensionFileCountMetaKey)); - Assert.Equal("[]", ReadMetaValue(extractedDb, DbContext.UnknownExtensionFilePathsMetaKey)); + Assert.Null(ReadMetaValue(extractedDb, DbContext.UnknownExtensionFileCountMetaKey)); + Assert.Null(ReadMetaValue(extractedDb, DbContext.UnknownExtensionFilePathsMetaKey)); AssertPartialArchiveStatus(extractedDb); var importedDb = Path.Combine(projectRoot, "imported", "codeindex.db"); @@ -1067,6 +1072,42 @@ public void RunExportArchive_AppliesProjectPathLanguageAndTestScope_Issue4714() Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(importedDb)))); AssertPartialArchiveStatus(importedDb); + var (refreshExitCode, refreshStdout, _) = ConsoleCapture.Capture(() => + ProgramRunner.Run( + [ + "index", projectRoot, + "--files", "src/App/App.cs", + "--db", importedDb, + "--json", + "--no-progress", + "--notify", "none", + ], + appVersion: "test")); + Assert.Equal(CommandExitCodes.Success, refreshExitCode); + using (var refreshResult = JsonDocument.Parse(refreshStdout)) + { + Assert.Equal(6, refreshResult.RootElement + .GetProperty("summary") + .GetProperty("files_total") + .GetInt32()); + Assert.True(refreshResult.RootElement.GetProperty("index_complete").GetBoolean()); + } + Assert.Equal("complete", ReadMetaValue(importedDb, DbContext.IndexCompletenessMetaKey)); + Assert.Null(ReadMetaValue(importedDb, DbContext.IndexIncompleteReasonsMetaKey)); + + var (refreshedStatusExitCode, refreshedStatusStdout, refreshedStatusStderr) = + ConsoleCapture.Capture(() => + ProgramRunner.Run( + ["status", "--check", "--db", importedDb, "--json"], + appVersion: "test")); + Assert.Equal(CommandExitCodes.Success, refreshedStatusExitCode); + Assert.Equal(string.Empty, refreshedStatusStderr); + using (var refreshedStatus = JsonDocument.Parse(refreshedStatusStdout)) + { + Assert.True(refreshedStatus.RootElement.GetProperty("index_complete").GetBoolean()); + Assert.True(refreshedStatus.RootElement.GetProperty("index_matches_workspace").GetBoolean()); + } + using var manifest = ZipFile.OpenRead(archivePath); using var manifestStream = manifest.GetEntry("manifest.json")!.Open(); using var manifestDocument = JsonDocument.Parse(manifestStream); @@ -1148,13 +1189,15 @@ public void RunImport_FullAndLegacyArchivesMaterializeConservativeTrust_Issue505 Assert.True( !legacyScope.TryGetProperty("represents_entire_source_database", out var wholeSource) || wholeSource.ValueKind == JsonValueKind.Null); - Assert.Equal(0, legacyResult.RootElement.GetProperty("unknown_extension_file_count").GetInt64()); + Assert.Equal( + JsonValueKind.Null, + legacyResult.RootElement.GetProperty("unknown_extension_file_count").ValueKind); } Assert.Equal("incomplete", ReadMetaValue(legacyImportDbPath, DbContext.IndexCompletenessMetaKey)); Assert.Null(ReadMetaValue(legacyImportDbPath, DbContext.IndexedHeadShaMetaKey)); Assert.Null(ReadMetaValue(legacyImportDbPath, DbContext.CommitScopedFreshHeadShaMetaKey)); Assert.Null(ReadMetaValue(legacyImportDbPath, DbContext.LastIndexRunFilesScannedMetaKey)); - Assert.Equal("0", ReadMetaValue(legacyImportDbPath, DbContext.UnknownExtensionFileCountMetaKey)); + Assert.Null(ReadMetaValue(legacyImportDbPath, DbContext.UnknownExtensionFileCountMetaKey)); Assert.Equal( "[\"legacy_source_incomplete\",\"partial_archive\"]", ReadMetaValue(legacyImportDbPath, DbContext.IndexIncompleteReasonsMetaKey)); @@ -2408,7 +2451,7 @@ private static void AssertPartialArchiveStatus(string dbPath) Assert.True( !status.RootElement.TryGetProperty("head_freshness", out var headFreshness) || headFreshness.GetProperty("state").GetString() != "head_current"); - Assert.Equal(0, status.RootElement.GetProperty("unknown_extension_file_count").GetInt64()); + Assert.False(status.RootElement.TryGetProperty("unknown_extension_file_count", out _)); Assert.True( !status.RootElement.TryGetProperty("last_index_run", out var lastIndexRun) || lastIndexRun.ValueKind == JsonValueKind.Null); From 360acaa448357482639f331e6e8af8d9d24dbcf1 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 10 Aug 2026 12:01:12 +0900 Subject: [PATCH 4/4] Release archive test database before hashing (#5053) --- .../ExportImportCommandRunnerTests.cs | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index 4dd40728c..020bc6bb2 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -1027,22 +1027,25 @@ public void RunExportArchive_AppliesProjectPathLanguageAndTestScope_Issue4714_Is var extractedDb = Path.Combine(projectRoot, "scoped.db"); using (var archive = ZipFile.OpenRead(archivePath)) archive.GetEntry("codeindex.db")!.ExtractToFile(extractedDb); - using var connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = extractedDb }.ConnectionString); - connection.Open(); - using (var filesCommand = connection.CreateCommand()) - { - filesCommand.CommandText = "SELECT path FROM files ORDER BY path"; - using var reader = filesCommand.ExecuteReader(); - var paths = new List(); - while (reader.Read()) - paths.Add(reader.GetString(0)); - Assert.Equal(["src/App/App.cs", "src/shared/Shared.cs"], paths); - } - using (var integrityCommand = connection.CreateCommand()) + using (var connection = new SqliteConnection( + new SqliteConnectionStringBuilder { DataSource = extractedDb }.ConnectionString)) { - integrityCommand.CommandText = "PRAGMA foreign_key_check"; - using var reader = integrityCommand.ExecuteReader(); - Assert.False(reader.Read()); + connection.Open(); + using (var filesCommand = connection.CreateCommand()) + { + filesCommand.CommandText = "SELECT path FROM files ORDER BY path"; + using var reader = filesCommand.ExecuteReader(); + var paths = new List(); + while (reader.Read()) + paths.Add(reader.GetString(0)); + Assert.Equal(["src/App/App.cs", "src/shared/Shared.cs"], paths); + } + using (var integrityCommand = connection.CreateCommand()) + { + integrityCommand.CommandText = "PRAGMA foreign_key_check"; + using var reader = integrityCommand.ExecuteReader(); + Assert.False(reader.Read()); + } } Assert.Equal("incomplete", ReadMetaValue(extractedDb, DbContext.IndexCompletenessMetaKey));