From ae3c3e115f577f8fc3db7083d02acc6089f5d252 Mon Sep 17 00:00:00 2001 From: chaunce Date: Wed, 4 Nov 2026 17:14:57 +0800 Subject: [PATCH] fix(pipeline): bound cross-file preparation memory Size C# and JVM scratch arrays by matching definitions. Refuse global cross-file allocations over the remaining budget and propagate budget aborts without replacing the serving index. Add self-contained allocation and publication-preservation regressions. Signed-off-by: chaunce --- docs/INDEX_RESOURCE_LIMITS.md | 14 +++ internal/cbm/lsp/cs_lsp.c | 8 +- internal/cbm/lsp/java_lsp.c | 8 +- src/foundation/mem.c | 8 ++ src/foundation/mem.h | 4 + src/pipeline/pass_lsp_cross.c | 57 ++++++++--- src/pipeline/pass_lsp_cross.h | 7 +- src/pipeline/pipeline.c | 53 ++++++++-- src/pipeline/pipeline_incremental.c | 26 +++-- tests/test_mem.c | 19 ++++ tests/test_pipeline.c | 144 ++++++++++++++++++++++++++++ 11 files changed, 316 insertions(+), 32 deletions(-) diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index d76b162d76..ee7a10f88f 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -78,6 +78,20 @@ its measurement is skipped, because a total without a live member would under-report. On Linux the probe reads every `/proc//stat` to find the tree's members, so a `/proc` mounted with `hidepid=1` makes it fail closed. +## Internal cross-file memory budget + +The internal `CBM_MEM_BUDGET_MB` budget also applies when extraction results +are gathered for cross-file resolution. The collector checks the remaining +charged-memory budget before allocating the project-wide definition array and +periodically while copying definitions. A budget refusal aborts the staged +update and preserves the serving index. The parallel driver also checks the +budget between shared registry builders. + +C# and Java/Kotlin registry scratch arrays scale with definitions of their +own languages, rather than every definition in the repository. This avoids +large unrelated data files multiplying those temporary allocations. These +checks remain cooperative; they do not replace the process-tree watchdog. + ## Counting and failure semantics `index_max_files` counts a file only after it passes directory pruning, ignore diff --git a/internal/cbm/lsp/cs_lsp.c b/internal/cbm/lsp/cs_lsp.c index 266b846a3a..b05831f2ff 100644 --- a/internal/cbm/lsp/cs_lsp.c +++ b/internal/cbm/lsp/cs_lsp.c @@ -3939,8 +3939,12 @@ CBMTypeRegistry *cbm_cs_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i * NULL, which must not be mistaken for OOM — the empty registry is still * built, finalized, and shared. */ CBMLSPDef *cs = NULL; - if (def_count > 0) { - cs = (CBMLSPDef *)cbm_arena_alloc(arena, (size_t)def_count * sizeof(*cs)); + int cs_count = 0; + for (int i = 0; i < def_count; i++) { + cs_count += defs[i].lang == CBM_LANG_CSHARP; + } + if (cs_count > 0) { + cs = (CBMLSPDef *)cbm_arena_alloc(arena, (size_t)cs_count * sizeof(*cs)); if (!cs) return NULL; } diff --git a/internal/cbm/lsp/java_lsp.c b/internal/cbm/lsp/java_lsp.c index cf221f06e7..37c005b5bc 100644 --- a/internal/cbm/lsp/java_lsp.c +++ b/internal/cbm/lsp/java_lsp.c @@ -3850,8 +3850,12 @@ CBMTypeRegistry *cbm_java_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, * NULL, which must not be mistaken for OOM — the empty registry is still * built, finalized, and shared. */ CBMLSPDef *jvm = NULL; - if (def_count > 0) { - jvm = (CBMLSPDef *)cbm_arena_alloc(arena, (size_t)def_count * sizeof(*jvm)); + int jvm_count = 0; + for (int i = 0; i < def_count; i++) { + jvm_count += defs[i].lang == CBM_LANG_JAVA || defs[i].lang == CBM_LANG_KOTLIN; + } + if (jvm_count > 0) { + jvm = (CBMLSPDef *)cbm_arena_alloc(arena, (size_t)jvm_count * sizeof(*jvm)); if (!jvm) { return NULL; } diff --git a/src/foundation/mem.c b/src/foundation/mem.c index 4f74229e15..26f08d2ac7 100644 --- a/src/foundation/mem.c +++ b/src/foundation/mem.c @@ -649,6 +649,14 @@ bool cbm_mem_over_budget(void) { return charged > g_budget; } +bool cbm_mem_would_exceed_budget(size_t additional_bytes) { + if (g_budget == 0) { + return false; + } + size_t charged = cbm_mem_charged(); + return charged > g_budget || additional_bytes > g_budget - charged; +} + /* "Give memory back now" — true when our own budget is exceeded OR the MACHINE * is short, whichever comes first. * diff --git a/src/foundation/mem.h b/src/foundation/mem.h index 91a8ef57bb..822cc27a20 100644 --- a/src/foundation/mem.h +++ b/src/foundation/mem.h @@ -104,6 +104,10 @@ void cbm_mem_set_charged_for_tests(size_t bytes); /* Returns true if current RSS exceeds the budget. */ bool cbm_mem_over_budget(void); +/* Reject a known upcoming allocation before touching its pages. A zero + * budget means initialization has not run yet. Overflow-safe. */ +bool cbm_mem_would_exceed_budget(size_t additional_bytes); + /* True when memory should be handed back NOW: our budget is exceeded, or the * machine itself is short. Use this for spill/reclaim decisions; use * cbm_mem_over_budget() for the decision to stop a run. System pressure must diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index a0a769fd12..2589de73b6 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -17,6 +17,7 @@ #include "pipeline/lsp_surface.h" #include "foundation/mem_core.h" +#include "foundation/mem.h" #include "result_spill.h" #include "pipeline/pipeline_internal.h" #include "pipeline/lsp_resolve.h" @@ -40,6 +41,7 @@ #include #include +#include #include #include #include @@ -1186,14 +1188,15 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are int *out_count, int *out_def_starts) { const cbm_result_spill_t *spill = ctx ? ctx->spill : NULL; bool owner_walk_required = ctx && atomic_load(&ctx->test_definition_owners_seen); - int total = 0; + CBMLSPDef *defs = NULL; + size_t total = 0; for (int i = 0; i < file_count; i++) { - int defs = 0; + int file_defs = 0; int impls = 0; int records = 0; /* a Python module's namespace records (re-exports) */ if (cache[i]) { owner_walk_required = owner_walk_required || cache[i]->has_test_definition_owners; - defs = cache[i]->defs.count; + file_defs = cache[i]->defs.count; impls = cache[i]->impl_traits.count; records = pxc_py_namespace_record_bound(cache[i]); } else if (spill && cbm_result_spill_has(spill, i)) { @@ -1201,12 +1204,12 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are * size), so the bound needs no full load. */ CBMFileResult hdr; if (cbm_result_spill_peek_header(spill, i, &hdr)) { - defs = hdr.defs.count; + file_defs = hdr.defs.count; impls = hdr.impl_traits.count; records = pxc_py_namespace_record_bound(&hdr); } } - total += defs; + total += file_defs; if (files[i].language == CBM_LANG_RUST) { total += impls; } @@ -1221,7 +1224,14 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are } return NULL; } - CBMLSPDef *defs = (CBMLSPDef *)calloc((size_t)total, sizeof(CBMLSPDef)); + /* Spilling extraction results does not bound this project-wide array or + * its owned strings. Refuse the lower bound before calloc can turn a + * data archive into several more gigabytes of anonymous memory. */ + if (total > INT_MAX || total > SIZE_MAX / sizeof(CBMLSPDef) || + cbm_mem_would_exceed_budget(total * sizeof(CBMLSPDef))) { + goto over_budget; + } + defs = (CBMLSPDef *)calloc(total, sizeof(CBMLSPDef)); if (!defs) { *out_count = 0; if (out_def_starts) { @@ -1232,6 +1242,9 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are int capacity = total; int idx = 0; for (int fi = 0; fi < file_count; fi++) { + if (cbm_mem_would_exceed_budget(0)) { + goto over_budget; + } if (out_def_starts) { out_def_starts[fi] = idx; } @@ -1251,12 +1264,8 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are #endif fr_loaded = fr != NULL; if (!fr && owner_walk_required) { - free(defs); *out_count = -1; - if (out_def_starts) { - memset(out_def_starts, 0, (size_t)(file_count + 1) * sizeof(int)); - } - return NULL; + goto collect_failure; } } if (!fr) @@ -1289,6 +1298,15 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are &bs.imp_count); } for (int di = 0; di < fr->defs.count; di++) { + /* One file can contain millions of JSON entries; a per-file + * check alone cannot bound the strings copied into the arena. */ + if ((di & 255) == 0 && cbm_mem_would_exceed_budget(0)) { + cbm_pxc_free_import_map(bs.imp_keys, bs.imp_vals, bs.imp_count); + if (fr_loaded) { + cbm_free_result(fr); + } + goto over_budget; + } if (pxc_build_lsp_def(arena, &fr->defs.items[di], def_modules[fi], namespace_name, files[fi].language, &defs[idx], bs.reg ? &bs : NULL) == 0) { idx++; @@ -1330,6 +1348,16 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are } *out_count = idx; return defs; + +over_budget: + cbm_log_error("mem.budget.exceeded", "phase", "lsp_cross_collect", "action", "fail_whole"); + *out_count = CBM_PXC_DEFS_OVER_BUDGET; +collect_failure: + free(defs); + if (out_def_starts) { + memset(out_def_starts, 0, ((size_t)file_count + 1) * sizeof(int)); + } + return NULL; } /* Return the one source import path bound to `local_name`, or NULL when the @@ -2547,7 +2575,10 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * cbm_pxc_collect_all_defs(ctx, &ctx->seq_cross_arena, cache, files, file_count, ctx->project_name, def_modules, &def_count, def_starts); if (def_count < 0 || (owner_walk_required && (!all_defs || def_count <= 0))) { - atomic_store(&ctx->test_declarations_failed, 1); + bool over_budget = def_count == CBM_PXC_DEFS_OVER_BUDGET; + if (!over_budget) { + atomic_store(&ctx->test_declarations_failed, 1); + } cbm_free(CBM_MEM_CLASS_EXTRACT, def_starts); free(all_defs); for (int i = 0; i < file_count; i++) { @@ -2558,7 +2589,7 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * cbm_pxc_set_rust_manifest(NULL); cbm_arena_destroy(&cargo_arena); } - return CBM_PIPELINE_ABORT_PRESERVE_DB; + return over_budget ? CBM_PIPELINE_ABORT_OVER_BUDGET : CBM_PIPELINE_ABORT_PRESERVE_DB; } /* Same seam as the parallel driver: serialize per-file surfaces while the * result cache is alive. Failure only degrades to a full rebuild on the diff --git a/src/pipeline/pass_lsp_cross.h b/src/pipeline/pass_lsp_cross.h index 51964bc41e..021abd3e47 100644 --- a/src/pipeline/pass_lsp_cross.h +++ b/src/pipeline/pass_lsp_cross.h @@ -43,13 +43,18 @@ /* True iff this language has a cbm_run_X_lsp_cross resolver wired up. */ bool cbm_pxc_has_cross_lsp(CBMLanguage lang); +/* Collector failure distinct from missing configured test-owner definitions. + * Every driver must fail the attempt, preserving the serving generation. */ +#define CBM_PXC_DEFS_OVER_BUDGET (-2) + /* Collect a project-wide CBMLSPDef[] from every cached file result. * def_modules[i] receives the module QN for files[i] (malloc'd; the * caller frees each entry then the array). String fields in the * returned CBMLSPDef[] are borrowed from cache[i]->arena and from * def_modules[i] — caller must keep both alive while the array is in * use. Returns the malloc'd array (free() it) and writes the entry - * count to *out_count. Returns NULL on alloc failure or when no defs + * count to *out_count. A budget refusal returns NULL with + * *out_count = CBM_PXC_DEFS_OVER_BUDGET. Returns NULL on alloc failure or when no defs * exist. out_def_starts (optional, file_count + 1 entries, caller-owned) * receives per-file prefix offsets: file i's defs occupy * [out_def_starts[i], out_def_starts[i+1]) — the LSP-surface serializer diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 3f61e122d3..5dd8dfd8f3 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -2894,7 +2894,10 @@ static int run_sequential_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, for (int si = 0; si < PL_SEQ_PASSES && rc == 0; si++) { cbm_clock_gettime(CLOCK_MONOTONIC, t); int pr = seq_passes[si].fn(ctx, files, file_count); - if (pr == CBM_PIPELINE_ABORT_PRESERVE_DB || atomic_load(&ctx->test_declarations_failed)) { + if (pr == CBM_PIPELINE_ABORT_OVER_BUDGET) { + rc = pr; + } else if (pr == CBM_PIPELINE_ABORT_PRESERVE_DB || + atomic_load(&ctx->test_declarations_failed)) { rc = CBM_PIPELINE_ABORT_PRESERVE_DB; } else if (pr != 0 && !seq_passes[si].ignore_err) { rc = pr; @@ -3051,6 +3054,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, int def_count = 0; CBMLSPDef *all_defs = NULL; int *def_starts = NULL; + CBMModuleDefIndex *module_def_index = NULL; /* The collected defs own their strings in this arena (results may be on * disk by now); the per-language cross registries share it. */ CBMArena cross_lsp_arena; @@ -3064,6 +3068,10 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, def_modules, &def_count, def_starts) : NULL; } + if (def_count == CBM_PXC_DEFS_OVER_BUDGET) { + rc = CBM_PIPELINE_ABORT_OVER_BUDGET; + goto cross_lsp_cleanup; + } if (def_count < 0) { atomic_store(&ctx->test_declarations_failed, 1); def_count = 0; /* keep metric counts valid; the fatal latch survives */ @@ -3085,7 +3093,6 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, } } CBM_PROF_END_N("lsp_cross_prepare", "2_surface_rows", t_surfaces, file_count); - free(def_starts); /* Build inverted index: module_qn → defs. The fused resolve_worker * uses this to filter the global all_defs[] down to just the defs * each file actually needs (own_module + imported modules) — the @@ -3093,8 +3100,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, * cost from O(all_defs) to O(relevant_defs), typically 50-100× * smaller per file. */ CBM_PROF_START(t_module_index); - CBMModuleDefIndex *module_def_index = - all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL; + module_def_index = all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL; CBM_PROF_END_N("lsp_cross_prepare", "3_module_def_index", t_module_index, def_count); /* Tier 2 full: pre-build per-language cross-LSP registries. * Built ONCE here; shared READ-ONLY across all files of that language @@ -3102,6 +3108,10 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, * — no registry build, no Phase 1b mutations. Languages added so far: * Go, Python, C/C++, C#, TS/JS, Java. Others (Kotlin, PHP) fall back to per-file. */ CBMCrossLspRegistries cross_registries = {0}; + if (cbm_mem_would_exceed_budget(0)) { + rc = CBM_PIPELINE_ABORT_OVER_BUDGET; + goto cross_lsp_cleanup; + } if (all_defs) { /* Per-builder split of lsp_cross_prepare — attributes a slow prepare to * ONE language instead of re-diagnosing the whole pass (the cs builder @@ -3110,23 +3120,47 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, long b_ms[6]; cbm_clock_gettime(CLOCK_MONOTONIC, &t_b); cross_registries.go = cbm_go_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + if (cbm_mem_would_exceed_budget(0)) { + rc = CBM_PIPELINE_ABORT_OVER_BUDGET; + goto cross_lsp_cleanup; + } b_ms[0] = (long)elapsed_ms(t_b); cbm_clock_gettime(CLOCK_MONOTONIC, &t_b); cross_registries.python = cbm_py_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + if (cbm_mem_would_exceed_budget(0)) { + rc = CBM_PIPELINE_ABORT_OVER_BUDGET; + goto cross_lsp_cleanup; + } b_ms[1] = (long)elapsed_ms(t_b); cbm_clock_gettime(CLOCK_MONOTONIC, &t_b); cross_registries.c = cbm_c_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + if (cbm_mem_would_exceed_budget(0)) { + rc = CBM_PIPELINE_ABORT_OVER_BUDGET; + goto cross_lsp_cleanup; + } b_ms[2] = (long)elapsed_ms(t_b); cbm_clock_gettime(CLOCK_MONOTONIC, &t_b); cross_registries.cs = cbm_cs_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + if (cbm_mem_would_exceed_budget(0)) { + rc = CBM_PIPELINE_ABORT_OVER_BUDGET; + goto cross_lsp_cleanup; + } b_ms[3] = (long)elapsed_ms(t_b); cbm_clock_gettime(CLOCK_MONOTONIC, &t_b); cross_registries.ts = cbm_ts_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + if (cbm_mem_would_exceed_budget(0)) { + rc = CBM_PIPELINE_ABORT_OVER_BUDGET; + goto cross_lsp_cleanup; + } b_ms[4] = (long)elapsed_ms(t_b); cbm_clock_gettime(CLOCK_MONOTONIC, &t_b); cross_registries.java = cbm_java_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + if (cbm_mem_would_exceed_budget(0)) { + rc = CBM_PIPELINE_ABORT_OVER_BUDGET; + goto cross_lsp_cleanup; + } b_ms[5] = (long)elapsed_ms(t_b); char b_buf[6][CBM_SZ_16]; const char *b_name[6] = {"go", "python", "c", "cs", "ts", "java"}; @@ -3157,6 +3191,11 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, itoa_buf((int)elapsed_ms(*t))); pipeline_phase_mark("parallel_resolve"); log_result_arenas("post_resolve", cache, file_count); +cross_lsp_cleanup: + if (rc == CBM_PIPELINE_ABORT_OVER_BUDGET) { + cbm_log_error("mem.budget.exceeded", "phase", "lsp_cross_prepare", "action", "fail_whole"); + } + free(def_starts); cbm_pxc_free_module_def_index(module_def_index); cbm_arena_destroy(&cross_lsp_arena); /* releases all per-lang registries */ free(all_defs); @@ -3167,8 +3206,10 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, free(def_modules); } cbm_pipeline_sync_next_id(p->gbuf, &shared_ids); - cbm_pipeline_extract_infra_routes(ctx, p->gbuf, files, cache, file_count); - cbm_pipeline_process_infra_bindings(ctx, p->gbuf, files, cache, file_count); + if (rc == 0) { + cbm_pipeline_extract_infra_routes(ctx, p->gbuf, files, cache, file_count); + cbm_pipeline_process_infra_bindings(ctx, p->gbuf, files, cache, file_count); + } for (int i = 0; i < file_count; i++) { if (cache[i]) { cbm_free_result(cache[i]); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 4e96e732af..10d1ce400f 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1955,8 +1955,12 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed closure->def_modules = def_modules; closure->def_module_count = def_modules ? ci : 0; def_modules = NULL; - atomic_store(&ctx->test_declarations_failed, 1); - rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + if (fresh_count == CBM_PXC_DEFS_OVER_BUDGET) { + rc = CBM_PIPELINE_ABORT_OVER_BUDGET; + } else { + atomic_store(&ctx->test_declarations_failed, 1); + rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + } goto cross_resolve_cleanup; } if ((fresh_defs || fresh_count == 0) && def_starts && @@ -2030,8 +2034,12 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed ctx->project_name, def_modules, &all_def_count, NULL); } if (!all_defs || all_def_count <= 0) { - atomic_store(&ctx->test_declarations_failed, 1); - rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + if (all_def_count == CBM_PXC_DEFS_OVER_BUDGET) { + rc = CBM_PIPELINE_ABORT_OVER_BUDGET; + } else { + atomic_store(&ctx->test_declarations_failed, 1); + rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + } goto cross_resolve_cleanup; } module_def_index = cbm_pxc_build_module_def_index(all_defs, all_def_count); @@ -2357,10 +2365,12 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project, /* Extract left pipeline NULL so file errors are not recorded twice. * Surface rows only borrow it for the background worker policy. */ probe_ctx.pipeline = p; - rc = def_count < 0 ? CBM_PIPELINE_ABORT_PRESERVE_DB - : cbm_lsp_surface_build_rows(&probe_ctx, project, cache, probe_files, - probe_count, defs, def_starts, out_rows, - out_count); + rc = def_count < 0 + ? (def_count == CBM_PXC_DEFS_OVER_BUDGET ? CBM_PIPELINE_ABORT_OVER_BUDGET + : CBM_PIPELINE_ABORT_PRESERVE_DB) + : cbm_lsp_surface_build_rows(&probe_ctx, project, cache, probe_files, + probe_count, defs, def_starts, out_rows, + out_count); probe_ctx.pipeline = NULL; } else { rc = -1; diff --git a/tests/test_mem.c b/tests/test_mem.c index ea4dd295ea..ad6344cecc 100644 --- a/tests/test_mem.c +++ b/tests/test_mem.c @@ -1737,6 +1737,24 @@ TEST(mem_over_budget_follows_the_charged_reading) { PASS(); } +TEST(mem_upcoming_allocation_respects_remaining_budget) { + size_t saved = cbm_mem_budget(); + cbm_mem_set_budget_for_tests(1024); + cbm_mem_set_charged_for_tests(1000); + bool exact_fits = !cbm_mem_would_exceed_budget(24); + bool too_big = cbm_mem_would_exceed_budget(25); + bool overflow = cbm_mem_would_exceed_budget(SIZE_MAX); + cbm_mem_set_charged_for_tests(1025); + bool already_over = cbm_mem_would_exceed_budget(0); + cbm_mem_set_charged_for_tests(0); + cbm_mem_set_budget_for_tests(saved); + ASSERT_TRUE(exact_fits); + ASSERT_TRUE(too_big); + ASSERT_TRUE(overflow); + ASSERT_TRUE(already_over); + PASS(); +} + TEST(mem_core_class_names_are_total) { ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_SEMANTIC), "semantic") == 0); ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_ARENA), "arena") == 0); @@ -1840,4 +1858,5 @@ SUITE(mem) { RUN_TEST(mem_system_available_ram_is_within_total); RUN_TEST(mem_system_under_pressure_is_a_pure_threshold); RUN_TEST(mem_over_budget_follows_the_charged_reading); + RUN_TEST(mem_upcoming_allocation_respects_remaining_budget); } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 8e4686b0e2..bf89aa8e5a 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -28664,6 +28664,147 @@ static unsigned char *read_file_bytes(const char *path, size_t *len_out) { return buf; } +/* An archive's unrelated definitions must not size C#/JVM scratch copies. + * Keep the same matching defs and compare the real arena capacity, rather + * than relying on RSS or a machine being short of memory. */ +TEST(pipeline_cross_registry_ignores_unrelated_allocation_size) { + enum { NOISE = 50000, MATCHES = 3 }; + CBMLSPDef *defs = calloc(NOISE + MATCHES, sizeof(*defs)); + ASSERT_NOT_NULL(defs); + for (int i = 0; i < NOISE; i++) { + defs[i] = (CBMLSPDef){.lang = CBM_LANG_JSON, + .qualified_name = "archive.entry", + .short_name = "entry", + .label = "Variable"}; + } + const CBMLanguage langs[] = {CBM_LANG_CSHARP, CBM_LANG_JAVA}; + size_t small[2] = {0}, mixed[2] = {0}; + bool valid = true; + for (int li = 0; li < 2; li++) { + /* A method before its type also exercises the stable type-first split. */ + defs[NOISE] = (CBMLSPDef){.lang = langs[li], + .label = "Method", + .qualified_name = "example.Widget.run", + .short_name = "run", + .receiver_type = "example.Widget"}; + defs[NOISE + 1] = (CBMLSPDef){.lang = langs[li], + .label = "Class", + .qualified_name = "example.Widget", + .short_name = "Widget"}; + defs[NOISE + 2] = (CBMLSPDef){.lang = CBM_LANG_KOTLIN, + .label = "Class", + .qualified_name = "example.KWidget", + .short_name = "KWidget"}; + for (int big = 0; big < 2; big++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMLSPDef *slice = big ? defs : defs + NOISE; + int count = big ? NOISE + MATCHES : MATCHES; + CBMTypeRegistry *reg = li == 0 ? cbm_cs_build_cross_registry(&arena, slice, count) + : cbm_java_build_cross_registry(&arena, slice, count); + valid = valid && reg && cbm_registry_lookup_type(reg, "example.Widget"); + if (li == 1) { + valid = valid && reg && cbm_registry_lookup_type(reg, "example.KWidget"); + } + if (big) { + mixed[li] = cbm_arena_capacity(&arena); + } else { + small[li] = cbm_arena_capacity(&arena); + } + cbm_arena_destroy(&arena); + } + } + free(defs); + ASSERT_TRUE(valid); + ASSERT_EQ(mixed[0], small[0]); + ASSERT_EQ(mixed[1], small[1]); + PASS(); +} + +TEST(pipeline_cross_collect_refuses_oversized_array_before_reading_defs) { + /* No backing definition array: an attempted copy would crash under ASan. + * Metadata alone proves the proposed array cannot fit the remaining budget. */ + CBMFileResult result = {0}; + result.defs.count = 1000000; + CBMFileResult *cache[] = {&result}; + cbm_file_info_t files[] = {{.rel_path = "archive.json", .language = CBM_LANG_JSON}}; + char *modules[] = {NULL}; + int starts[] = {7, 9}; + int count = 0; + CBMArena arena; + cbm_arena_init(&arena); + size_t saved = cbm_mem_budget(); + cbm_mem_set_budget_for_tests(1024 * 1024); + cbm_mem_set_charged_for_tests(512 * 1024); + CBMLSPDef *defs = + cbm_pxc_collect_all_defs(NULL, &arena, cache, files, 1, "budget", modules, &count, starts); + cbm_mem_set_charged_for_tests(0); + cbm_mem_set_budget_for_tests(saved); + free(defs); + cbm_arena_destroy(&arena); + free(modules[0]); + ASSERT_NULL(defs); + ASSERT_EQ(count, CBM_PXC_DEFS_OVER_BUDGET); + ASSERT_EQ(starts[0], 0); + ASSERT_EQ(starts[1], 0); + PASS(); +} + +TEST(pipeline_cross_prepare_budget_preserves_serving_generation) { + if (!write_backpressure_fixture()) { + FAIL("failed to create fixture"); + } + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/cross-budget.db", g_tmpdir); + cbm_pipeline_t *p = cbm_pipeline_new(g_tmpdir, db_path, CBM_MODE_FAST); + int first = p ? cbm_pipeline_run(p) : -1; + cbm_pipeline_free(p); + size_t before_len = 0; + unsigned char *before = read_file_bytes(db_path, &before_len); + bool written = true; + for (int i = 0; i < 64; i++) { + char path[512]; + snprintf(path, sizeof(path), "%s/f%02d.go", g_tmpdir, i); + FILE *f = fopen(path, "w"); + if (!f) { + written = false; + break; + } + fprintf(f, "package main\n"); + for (int j = 0; j < 256; j++) { + fprintf(f, "func F%02d_%03d() int { return %d }\n", i, j, j); + } + fclose(f); + } + /* Extraction sees charge below budget, so its backpressure gate cannot + * cause this refusal. The cross-def array exceeds the remaining 1 MiB. */ + size_t saved = cbm_mem_budget(); + cbm_mem_set_budget_for_tests(16 * 1024 * 1024); + cbm_mem_set_charged_for_tests(15 * 1024 * 1024); + cbm_pp_bp_nap_cycles_reset(); + p = cbm_pipeline_new(g_tmpdir, db_path, CBM_MODE_FAST); + int second = p ? cbm_pipeline_run(p) : -1; + long cycles = cbm_pp_bp_nap_cycles(); + cbm_mem_set_charged_for_tests(0); + cbm_mem_set_budget_for_tests(saved); + cbm_pipeline_free(p); + size_t after_len = 0; + unsigned char *after = read_file_bytes(db_path, &after_len); + bool same = + before && after && before_len == after_len && memcmp(before, after, before_len) == 0; + int stages = count_generation_stage_artifacts(g_tmpdir, "cross-budget.db"); + free(before); + free(after); + teardown_test_repo(); + ASSERT_EQ(first, 0); + ASSERT_TRUE(written); + ASSERT_EQ(second, CBM_PIPELINE_ABORT_OVER_BUDGET); + ASSERT_EQ(cycles, 0); + ASSERT_TRUE(same); + ASSERT_EQ(stages, 0); + PASS(); +} + TEST(pipeline_backpressure_futile_nap_disengages) { /* 64 tiny files: > MIN_FILES_FOR_PARALLEL (50) so the parallel path (and its * back-pressure gate) actually runs; old-code cycles (~64) >> the bound. */ @@ -31798,6 +31939,9 @@ SUITE(pipeline) { RUN_TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing); RUN_TEST(pipeline_late_source_limit_preserves_previous_generation); /* Extraction back-pressure */ + RUN_TEST(pipeline_cross_registry_ignores_unrelated_allocation_size); + RUN_TEST(pipeline_cross_collect_refuses_oversized_array_before_reading_defs); + RUN_TEST(pipeline_cross_prepare_budget_preserves_serving_generation); RUN_TEST(pipeline_backpressure_futile_nap_disengages); RUN_TEST(pipeline_over_budget_after_futility_fails_whole_and_preserves_db); /* Sequential cross-LSP shared registry (ms-typescript quadratic) */