From 4191342071a381b956b226620c79b29562d5f45d Mon Sep 17 00:00:00 2001 From: RetroGameDeveloper <40120498+RetroGameDeveloper@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:33:24 +0000 Subject: [PATCH 1/2] Avoid invalid repair compiles and resume reuse searches fairly --- README.md | 17 +++ ROADMAP.md | 7 + src/decompnet/app/compiler_preflight.py | 40 ++++++ src/decompnet/app/deep_permuter.py | 19 ++- src/decompnet/app/draft_repairs.py | 3 + src/decompnet/app/engine/briefs.py | 8 +- src/decompnet/app/free_candidates.py | 165 +++++++++++++----------- src/decompnet/app/local_experiments.py | 2 + src/decompnet/app/permute.py | 3 + src/decompnet/app/repair_preflight.py | 68 ++++++++++ src/decompnet/app/reuse_donors.py | 40 ++++++ src/decompnet/app/reuse_traversal.py | 27 ++++ src/decompnet/app/similar_matches.py | 16 +-- src/decompnet/toolchains.py | 7 + tests/test_compiler_preflight.py | 50 +++++++ tests/test_deep_permuter.py | 12 ++ tests/test_free_matching.py | 26 ++++ tests/test_local_experiments.py | 2 +- tests/test_repair_preflight.py | 36 ++++++ tests/test_reuse_donors.py | 29 +++++ tests/test_reuse_traversal.py | 23 ++++ 21 files changed, 508 insertions(+), 92 deletions(-) create mode 100644 src/decompnet/app/compiler_preflight.py create mode 100644 src/decompnet/app/repair_preflight.py create mode 100644 src/decompnet/app/reuse_donors.py create mode 100644 src/decompnet/app/reuse_traversal.py create mode 100644 tests/test_compiler_preflight.py create mode 100644 tests/test_repair_preflight.py create mode 100644 tests/test_reuse_donors.py create mode 100644 tests/test_reuse_traversal.py diff --git a/README.md b/README.md index 32978b6f..2cc984eb 100644 --- a/README.md +++ b/README.md @@ -621,6 +621,23 @@ Verified full-image imports include linked assembly routines as target-only unit as recovered C. Assembly wrappers containing whole binary programs are not treated as single functions; an explicit coverage notice identifies their exclusion from function progress. +### Repair preflight and reuse discovery + +Saved ROM drafts get an advisory warning preflight in agent briefs and deep-search output when their pinned +GCC supports it. It uses separate warning flags and caches diagnostics for unchanged compiler, header and +source inputs. An uninitialized-variable warning is a reason to check the draft before tuning it; warnings +never change project settings or count as a match. Imported projects retain diagnostics from their own build; +extra diagnostic builds for those projects and other compiler families are not supported yet. + +Generated local repairs skip direct calls that disagree with an unambiguous plain C prototype. Macros, +includes, indirect calls, unspecified argument lists, variable arguments and C++ remain the compiler's job. +The normal compiler and exact judge still decide every accepted improvement. + +Template/address/SDK discovery remembers its last visited function, including interrupted passes. Cached +misses are read together per function, and this phase takes at most half the remaining reuse time so wider +cross-project discovery can run too. Both phases retain the shared compile budget; a cursor does not skip +new candidates permanently. + ### Permuter budgets For a longer search of one function, open its drawer and expand **Deep permuter ยท this function**. diff --git a/ROADMAP.md b/ROADMAP.md index 7a525450..a7ae8e8b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -881,3 +881,10 @@ projects. It bypasses the background queue's size and turn filters, visits the g then uses ordinary random rewrites. Time/compile limits, progress, cancellation, normal judges and immediate saving remain mandatory. Scoped misses survive another deep run; failed compiles stay retryable and cached improvements must be rejudged. This does not claim to exhaust all possible C programs or compiler options. + +Repair preflight filters provable plain-prototype arity errors in shared generated local families and ROM +permuter/reuse candidates, before the ordinary judges. Saved ROM GCC drafts get cached advisory warnings in +agent briefs and deep-search output. Planned: compiler-specific diagnostic flags for other backends and an +opt-in diagnostic build command for imported projects (their ordinary build diagnostics remain available). +Reuse discovery batches fingerprint names/cache reads and persists a separate cheap-discovery cursor; at most +half the remaining shared reuse time goes to that phase, leaving time for wider matching. diff --git a/src/decompnet/app/compiler_preflight.py b/src/decompnet/app/compiler_preflight.py new file mode 100644 index 00000000..c76a606d --- /dev/null +++ b/src/decompnet/app/compiler_preflight.py @@ -0,0 +1,40 @@ +"""Advisory diagnostics for saved ROM drafts, separate from the exact match judge. + +A private N64 near miss used an uninitialized pointer. One warning-enabled compile per input snapshot +makes that evidence available before more permutations; it never supplies a score or changes project flags. +""" +import hashlib +import json +import logging + +from .. import toolchains +from ..project import compiler_for + +log = logging.getLogger('decompdecomp') + + +def warnings(net, proj, fn): + """Return cached diagnostics where the pinned compiler supports warning flags, without keeping its object.""" + if not fn['best_source'] or net.fork_slug(proj['slug']): + return '' # Imported projects own their build commands; their ordinary build retains its diagnostics. + cfg = net.cfg(proj) + settings = compiler_for(cfg, fn['vram']) + flags = toolchains.diagnostic_flags(settings.get('toolchain', '')) + if not flags or settings.get('cmd') or not net.is_rom(proj): + return '' + header = net._context(proj['id'], fn['unit'])['header'] + key = hashlib.sha256(repr((proj['root'], settings, header, fn['best_source'])).encode()).hexdigest() + cache = getattr(net, '_compiler_preflights', {}) + net._compiler_preflights = cache + if key not in cache: + diagnostic = {**cfg, 'compiler_ranges': [], 'compiler': {**settings, 'flags': [*settings.get('flags', []), *flags]}} + try: + result = net._compile({**dict(proj), 'config': json.dumps(diagnostic)}, header, fn['best_source'], fn) + lines = [line for line in result.log.splitlines() if 'warning' in line.lower() or 'uninitialized' in line.lower()] + cache[key] = '\n'.join(lines)[:1000] if result.ok else '' + except Exception: # advisory evidence must never stand between the user and the real judge + log.exception('compiler warning preflight failed') + return '' # Tool failures remain retryable. + while len(cache) > 256: + del cache[next(iter(cache))] + return cache[key] diff --git a/src/decompnet/app/deep_permuter.py b/src/decompnet/app/deep_permuter.py index 4a6af690..096ca351 100644 --- a/src/decompnet/app/deep_permuter.py +++ b/src/decompnet/app/deep_permuter.py @@ -40,7 +40,7 @@ def __init__(self, net, minutes, max_variants, log): self.seconds = minutes*60 self.limit = max_variants self.log = log - self.tested = self.skipped = self.visited = 0 + self.tested = self.skipped = self.visited = self.rejected = 0 self.best = None self.phase = 'local repairs' self.measuring_baseline = False @@ -54,7 +54,7 @@ def check(self): def report(self): self.log.progress({'phase': self.phase, 'tested': self.tested, 'skipped': self.skipped, - 'best_score': self.best, 'variant_limit': self.limit, + 'best_score': self.best, 'variant_limit': self.limit, 'rejected': self.rejected, 'seconds_left': max(0, round(self.deadline-time.monotonic()))}) def _baseline(self, *args): @@ -69,6 +69,13 @@ def _baseline(self, *args): def _score(self, agent, proj, fn, source, best, fork): self.check() self.visited += 1 + if not fork and not self.measuring_baseline and source != fn['best_source']: + from .repair_preflight import invalid_call + header = self.net._context(proj['id'], fn['unit'])['header'] + if invalid_call(source, fn['name'], header): + self.rejected += 1 + self.report() + return None # Scope includes the actual compiler/header/target and imported build dependencies. A previous # miss is reusable only in that scope. Better cached results are rejudged before being kept. scope = agent_tokens.target_scope(self.net, {**dict(fn), 'best_source': source}, '') @@ -97,6 +104,10 @@ def run(net, slug, name, minutes, max_variants, log): state = State() agent, _ = net.local_agent('permuter') reason = 'Current candidate families finished' + from .compiler_preflight import warnings + warning = warnings(net, proj, fn) + if warning: + log('Compiler preflight (advisory): '+warning) log('Testing supported local repairs, then bounded random rewrites. This cannot exhaust every possible C program.') try: if not search._hold(agent, fn, count_turn=False): @@ -145,6 +156,6 @@ def run(net, slug, name, minutes, max_variants, log): except SearchLimit as error: reason = str(error) search.report() - log(f'{reason}: {search.tested} candidates tested, {search.skipped} cached misses skipped. Verified improvements are saved immediately.') + log(f'{reason}: {search.tested} candidates tested, {search.skipped} cached misses skipped, {search.rejected} invalid calls skipped. Verified improvements are saved immediately.') return {'function': name, 'reason': reason, 'tested': search.tested, 'skipped': search.skipped, - 'matched': net._function(name, slug)['status'] == 'matched', 'best_score': search.best} + 'rejected': search.rejected, 'matched': net._function(name, slug)['status'] == 'matched', 'best_score': search.best} diff --git a/src/decompnet/app/draft_repairs.py b/src/decompnet/app/draft_repairs.py index 41e5694c..663359d5 100644 --- a/src/decompnet/app/draft_repairs.py +++ b/src/decompnet/app/draft_repairs.py @@ -148,7 +148,10 @@ def add(a, b, text): labels.setdefault(text,'expression_or_near_repair') # Round-robin families so a new family cannot crowd existing repairs out of a short pass. groups = {} + from .repair_preflight import invalid_call for text in dict.fromkeys(proposals): + if invalid_call(text, name): + continue groups.setdefault(labels[text],[]).append(text) ordered = [] while any(groups.values()): diff --git a/src/decompnet/app/engine/briefs.py b/src/decompnet/app/engine/briefs.py index c2a9daad..93130c89 100644 --- a/src/decompnet/app/engine/briefs.py +++ b/src/decompnet/app/engine/briefs.py @@ -247,7 +247,13 @@ def _brief(self, fn, agent, header): fn = self._function(fn["name"], proj["slug"]) if fn["seed"]: fn = dict(fn) | {"seed": self._draft_shown(fn, proj)} - return super()._brief(fn, agent, header) + text = super()._brief(fn, agent, header) + if fn['status'] == 'open' and fn['best_source']: + from .. import compiler_preflight + warning = compiler_preflight.warnings(self, proj, fn) + if warning: + text += '\n\nCompiler preflight (advisory; check the draft before tuning):\n' + warning + return text def xrefs(self, proj) -> dict: """calls / called_by / uses / used_by, all keyed by address.""" diff --git a/src/decompnet/app/free_candidates.py b/src/decompnet/app/free_candidates.py index 57f259cc..58e46138 100644 --- a/src/decompnet/app/free_candidates.py +++ b/src/decompnet/app/free_candidates.py @@ -87,7 +87,7 @@ def exhausted(job): def run(twins, slug, job): """Try recognisable leaf patterns and exact SDK fingerprints before spending CPU on wider similarity.""" - from . import repair_mappings + from . import repair_mappings, repair_preflight, reuse_traversal from .address_repairs import variants as address_variants from .sdk_library import allowed from .sdk_source_repairs import variant as sdk_variant @@ -106,86 +106,97 @@ def run(twins, slug, job): matched = 0 sdk_maps = {} by_fingerprint = {} - function_names = {r['vram']: r['name'] for r in net.store.all('SELECT vram,name FROM functions WHERE project_id=?', proj['id'])} + names = net.store.all('SELECT id,vram,name FROM functions WHERE project_id=?', proj['id']) + function_names = {r['vram']: r['name'] for r in names} + names_by_id = {r['id']: r['name'] for r in names} for fid, fingerprint in prints.items(): - row = net.store.one('SELECT name FROM functions WHERE id=?', fid) - if row: - by_fingerprint.setdefault(fingerprint.key, set()).add(row['name']) + if fid in names_by_id: + by_fingerprint.setdefault(fingerprint.key, set()).add(names_by_id[fid]) symbols = [(r['name'], r['address']) for r in net.store.all('SELECT name,address FROM symbols WHERE project_id=? ORDER BY name', proj['id'])] - for fn in net.store.all("SELECT * FROM functions WHERE project_id=? AND status='open' ORDER BY size,id", proj['id']): - if exhausted(job): - break - if net.store.one('SELECT 1 FROM claims WHERE function_id=?', fn['id']): - continue - fingerprint = prints.get(fn['id']) - if fingerprint is None or fingerprint.asm: - continue - settings = compiler_for(cfg, fn['vram']) - candidates = [('template', 'MIPS leaf template', code) for code in mips_templates.candidates(fn, bool(proj['big_endian']), function_names)] - if fn['best_source'] and fn['best_score'] is not None and 0 < fn['best_score'] <= (100 - REPAIR_PERCENT) * max(1, fn['n_insns'] or 0): - repair_mappings.recover(twins, proj, fn, fn['best_source']) - candidates.insert(0, ('address-probe', 'saved near miss', fn['best_source'])) - learned = repair_mappings.variant(twins, proj, fn, fn['best_source']) - if learned: - candidates.insert(0, ('learned-symbols', 'verified donor mappings', learned)) - sdk = net.store.all('SELECT DISTINCT s.* FROM sdk_sources s JOIN sdk_signatures i ' - 'ON s.corpus=i.corpus AND s.name=i.name AND s.arch=i.arch AND s.big_endian=i.big_endian ' - 'WHERE i.arch=? AND i.big_endian=? AND i.fingerprint=? AND s.compiler=? ORDER BY s.corpus,s.name', - proj['arch'], proj['big_endian'], fingerprint.key, twins._compiler_key(settings)) - sdk = [row for row in sdk if allowed(net, proj, row['corpus'])][:3] - for row in sdk: - corpus = row['corpus'] - if corpus not in sdk_maps: - signatures = net.store.all('SELECT name,fingerprint,size FROM sdk_signatures WHERE corpus=? AND arch=? AND big_endian=?', - corpus, proj['arch'], proj['big_endian']) - sdk_maps[corpus] = sdk_symbol_map(signatures, by_fingerprint) - names = {**sdk_maps[corpus], row['name']: fn['name']} - candidates.append(('library', corpus + '/' + row['name'], apply(row['header'] + '\n' + row['source'], names))) - for kind, origin, source in candidates: - if exhausted(job): - return matched - if kind in ('address-probe', 'address-repair', 'learned-symbols'): - repair_mappings.propagate(net, fn, fn['best_source'], source) - context = net._context(proj['id'], fn['unit'])['header'] - key = trial_key(source, context, settings, fn, [kind, symbols]) - previous = net.store.one("SELECT feedback FROM reuse_trials WHERE function_id=? AND signature=? AND outcome='miss'", fn['id'], key) - if previous: - if kind == 'library': + # Leave half of the remaining wall time for wider cross-project discovery. Cached template/address + # work must not prevent that phase from ever starting; both still share the original compile budget. + phase_deadline = time.monotonic() + max(0, job.reuse_deadline-time.monotonic()) / 2 + functions = net.store.all("SELECT * FROM functions WHERE project_id=? AND status='open' ORDER BY size,id", proj['id']) + with reuse_traversal.rotating(net, proj['id'], functions, + ready=lambda: not exhausted(job) and time.monotonic() < phase_deadline) as rows: + for fn in rows: + if exhausted(job) or time.monotonic() >= phase_deadline: + break + if net.store.one('SELECT 1 FROM claims WHERE function_id=?', fn['id']): + continue + fingerprint = prints.get(fn['id']) + if fingerprint is None or fingerprint.asm: + continue + settings = compiler_for(cfg, fn['vram']) + candidates = [('template', 'MIPS leaf template', code) for code in mips_templates.candidates(fn, bool(proj['big_endian']), function_names)] + if fn['best_source'] and fn['best_score'] is not None and 0 < fn['best_score'] <= (100 - REPAIR_PERCENT) * max(1, fn['n_insns'] or 0): + repair_mappings.recover(twins, proj, fn, fn['best_source']) + candidates.insert(0, ('address-probe', 'saved near miss', fn['best_source'])) + learned = repair_mappings.variant(twins, proj, fn, fn['best_source']) + if learned: + candidates.insert(0, ('learned-symbols', 'verified donor mappings', learned)) + sdk = net.store.all('SELECT DISTINCT s.* FROM sdk_sources s JOIN sdk_signatures i ' + 'ON s.corpus=i.corpus AND s.name=i.name AND s.arch=i.arch AND s.big_endian=i.big_endian ' + 'WHERE i.arch=? AND i.big_endian=? AND i.fingerprint=? AND s.compiler=? ORDER BY s.corpus,s.name', + proj['arch'], proj['big_endian'], fingerprint.key, twins._compiler_key(settings)) + sdk = [row for row in sdk if allowed(net, proj, row['corpus'])][:3] + for row in sdk: + corpus = row['corpus'] + if corpus not in sdk_maps: + signatures = net.store.all('SELECT name,fingerprint,size FROM sdk_signatures WHERE corpus=? AND arch=? AND big_endian=?', + corpus, proj['arch'], proj['big_endian']) + sdk_maps[corpus] = sdk_symbol_map(signatures, by_fingerprint) + names = {**sdk_maps[corpus], row['name']: fn['name']} + candidates.append(('library', corpus + '/' + row['name'], apply(row['header'] + '\n' + row['source'], names))) + misses = {r['signature']: r for r in net.store.all( + "SELECT signature,feedback FROM reuse_trials WHERE function_id=? AND outcome='miss' AND length(signature)=64", fn['id'])} + for kind, origin, source in candidates: + if exhausted(job) or time.monotonic() >= phase_deadline: + return matched + if kind in ('address-probe', 'address-repair', 'learned-symbols'): + repair_mappings.propagate(net, fn, fn['best_source'], source) + context = net._context(proj['id'], fn['unit'])['header'] + if repair_preflight.invalid_call(source, fn['name'], context): + continue + key = trial_key(source, context, settings, fn, [kind, symbols]) + previous = misses.get(key) + if previous: + if kind == 'library': + repaired = sdk_variant(net, proj, fn, source, fingerprint, job) + if repaired: + candidates.append(('library-repair', origin, repaired)) + if kind in ('address-probe', 'learned-symbols'): + candidates.extend(('address-repair', origin, v) for v in + address_variants(source, json.loads(previous['feedback']), fn, + bool(proj['big_endian']), {name for name, _ in symbols})) + continue + job.reuse_attempts += 1 + try: + with repair_trial({'kind': 'automatic candidate', 'family': kind, 'origin': origin}): + result = net.try_quietly(agent, proj, fn, source) + except NetworkError: + continue # busy work remains retryable + outcome = 'match' if result['matched'] else 'miss' + reason = origin + '; ' + ('byte match' if result['matched'] else f"score {result.get('score')}") + with net.store.tx() as db: + db.execute('INSERT OR REPLACE INTO reuse_trials(function_id,signature,outcome,reason,feedback) VALUES(?,?,?,?,?)', + (fn['id'], key, outcome, reason, json.dumps(result))) + db.execute('INSERT INTO free_match_attempts(project_id,function_id,strategy,outcome,reason,created_at) VALUES(?,?,?,?,?,?)', + (proj['id'], fn['id'], kind, outcome, reason, time.time())) + if kind == 'library' and not result['matched']: repaired = sdk_variant(net, proj, fn, source, fingerprint, job) if repaired: candidates.append(('library-repair', origin, repaired)) - if kind in ('address-probe', 'learned-symbols'): + if kind in ('address-probe', 'learned-symbols') and not result['matched']: candidates.extend(('address-repair', origin, v) for v in - address_variants(source, json.loads(previous['feedback']), fn, - bool(proj['big_endian']), {name for name, _ in symbols})) - continue - job.reuse_attempts += 1 - try: - with repair_trial({'kind': 'automatic candidate', 'family': kind, 'origin': origin}): - result = net.try_quietly(agent, proj, fn, source) - except NetworkError: - continue # busy work remains retryable - outcome = 'match' if result['matched'] else 'miss' - reason = origin + '; ' + ('byte match' if result['matched'] else f"score {result.get('score')}") - with net.store.tx() as db: - db.execute('INSERT OR REPLACE INTO reuse_trials(function_id,signature,outcome,reason,feedback) VALUES(?,?,?,?,?)', - (fn['id'], key, outcome, reason, json.dumps(result))) - db.execute('INSERT INTO free_match_attempts(project_id,function_id,strategy,outcome,reason,created_at) VALUES(?,?,?,?,?,?)', - (proj['id'], fn['id'], kind, outcome, reason, time.time())) - if kind == 'library' and not result['matched']: - repaired = sdk_variant(net, proj, fn, source, fingerprint, job) - if repaired: - candidates.append(('library-repair', origin, repaired)) - if kind in ('address-probe', 'learned-symbols') and not result['matched']: - candidates.extend(('address-repair', origin, v) for v in - address_variants(source, result, fn, bool(proj['big_endian']), - {name for name, _ in symbols})) - if result['matched']: - if kind in ('address-probe', 'address-repair', 'learned-symbols'): - repair_mappings.accepted(net, fn, source) - matched += 1 - job.reuse_matched += 1 - job.reuse_bytes += fn['size'] - net.add_note(agent, slug, 'Autopilot matched from ' + origin, fn['name']) - break + address_variants(source, result, fn, bool(proj['big_endian']), + {name for name, _ in symbols})) + if result['matched']: + if kind in ('address-probe', 'address-repair', 'learned-symbols'): + repair_mappings.accepted(net, fn, source) + matched += 1 + job.reuse_matched += 1 + job.reuse_bytes += fn['size'] + net.add_note(agent, slug, 'Autopilot matched from ' + origin, fn['name']) + break return matched diff --git a/src/decompnet/app/local_experiments.py b/src/decompnet/app/local_experiments.py index 487530c2..45752018 100644 --- a/src/decompnet/app/local_experiments.py +++ b/src/decompnet/app/local_experiments.py @@ -82,6 +82,8 @@ def run(net, agent, slug, function, revision, families=None, max_trials=6): if 'narrowing' in pools: masks=lambda text:len(re.findall(r'&\s*0x(?:[fF]{4}|[fF]{2})\b',text)) pools['narrowing'] += [s for s in expression_repairs.variants(source,fn['name'],proj['arch']) if masks(s) i res = self.net.forks.attempt(agent, fork, fn["name"], source, keep=False, quiet=best) return res.get("score") ctx = self.net._context(proj["id"], fn["unit"]) + from .repair_preflight import invalid_call + if source != fn['best_source'] and invalid_call(source, fn['name'], ctx['header']): + return None result = self.net._compile(proj, ctx["header"], source, fn) if not result.ok: return None diff --git a/src/decompnet/app/repair_preflight.py b/src/decompnet/app/repair_preflight.py new file mode 100644 index 00000000..be531214 --- /dev/null +++ b/src/decompnet/app/repair_preflight.py @@ -0,0 +1,68 @@ +"""Conservative checks before generated repairs spend a compiler invocation. + +This is not a C parser. Unknown declarations, macros, C++ and indirect calls stay with the compiler. +Only an unambiguous plain prototype can rule out a direct call's argument count. +""" +import re + +from ..forks.csource import _COMMENT_OR_STRING, find + +_IDENT = r'[A-Za-z_]\w*' +_TYPE = r'(?:void|char|short|int|long|float|double|signed|unsigned|const|volatile|struct\s+\w+|u(?:8|16|32|64)|s(?:8|16|32|64))' +_PROTO = re.compile(rf'(?:^|(?<=[;}}]))\s*(?:extern\s+|static\s+)?(?:{_TYPE}\s+|\*\s*)+({_IDENT})\s*\(([^(){{}};]*)\)\s*;', re.M) + + +def clean(text): + return _COMMENT_OR_STRING.sub(lambda m: '0' if m.group().startswith(('"', "'")) else ' ', text) + + +def arguments(text): + depth = 0 + count = 1 if text.strip() else 0 + for char in text: + depth += (char in '([{') - (char in ')]}') + if char == ',' and depth == 0: + count += 1 + return count + + +def invalid_call(source, name, header=''): + """Explain a provable arity error, otherwise let the ordinary compiler decide.""" + text = clean(header+'\n'+source) + # Conditional declarations and macro expansion need the real preprocessor. Even object macros can + # expand to commas, so skip rather than guess. Included files may define macros too. + if re.search(r'^\s*#', text, re.M) or re.search(r'::|\b(?:class|template|namespace)\b', text): + return '' + span = find(source, name) + if not span: + return '' + body = clean(source[span[0]:span[1]]) + body = body[body.find('{')+1:] + prototypes = {} + for match in _PROTO.finditer(text): + params = match[2].strip() + if not params or '...' in params or '=' in params: + prototypes.setdefault(match[1], set()).add(None) + continue # empty () is unspecified in C; default parameters/varargs are not fixed arity + if params != 'void' and any(not re.match(rf'^{_TYPE}\b', p.strip()) for p in params.split(',')): + prototypes.setdefault(match[1], set()).add(None) + continue # typedefs and old-style names may need macro expansion + prototypes.setdefault(match[1], set()).add(0 if params == 'void' else arguments(params)) + for call in re.finditer(rf'(?])({_IDENT})\s*\(', body): + if call[1] not in prototypes or len(prototypes[call[1]]) != 1 or None in prototypes[call[1]]: + continue + # A local object/parameter with this name can shadow the declaration; leave it to the compiler. + if re.search(rf'\b{re.escape(call[1])}\b(?!\s*\()', clean(source)): + continue + start = call.end() + depth, end = 1, start + while end < len(body) and depth: + depth += (body[end] == '(') - (body[end] == ')') + end += 1 + if depth: + continue + actual = arguments(body[start:end-1]) + expected = next(iter(prototypes[call[1]])) + if actual != expected: + return f'{call[1]} expects {expected} arguments; generated call has {actual}.' + return '' diff --git a/src/decompnet/app/reuse_donors.py b/src/decompnet/app/reuse_donors.py new file mode 100644 index 00000000..dcb94f11 --- /dev/null +++ b/src/decompnet/app/reuse_donors.py @@ -0,0 +1,40 @@ +"""Shortlist donor metadata once; read large function bodies only when a search needs them. + +The old per-target query repeatedly loaded hundreds of the same sources and target byte arrays even when +all their trials were cached misses. This cache lasts one reuse pass, so the next pass sees donor changes. +""" +from bisect import bisect_left, bisect_right +from collections import OrderedDict + +MAX_BODIES = 1024 + + +class Donors: + def __init__(self, net, proj): + self.net = net + self.rows = net.store.all( + 'SELECT i.function_id,i.size,i.compiler,i.fingerprint FROM cross_game_index i ' + "JOIN functions f ON f.id=i.function_id AND f.status='matched' " + 'JOIN projects p ON p.id=i.project_id WHERE i.project_id!=? AND p.arch=? AND p.big_endian=? ' + 'ORDER BY i.size,i.function_id', proj['id'], proj['arch'], proj['big_endian']) + self.sizes = [row['size'] for row in self.rows] + self.bodies = OrderedDict() + + def candidates(self, size, low, high, fingerprint, compiler, limit): + """Preserve the previous size/distance/id ordering and limit, without reloading cached source blobs.""" + rows = self.rows[bisect_left(self.sizes, low):bisect_right(self.sizes, high)] + rows = [row for row in rows if row['fingerprint'] != fingerprint or row['compiler'] != compiler] + rows.sort(key=lambda row: (abs(row['size']-size), row['function_id'])) + result = [] + for row in rows[:limit]: + fid = row['function_id'] + if fid not in self.bodies: + body = self.net.store.one("SELECT * FROM functions WHERE id=? AND status='matched'", fid) + self.bodies[fid] = dict(body) if body else None + self.bodies.move_to_end(fid) + body = self.bodies[fid] + if body is not None: + result.append({**body, 'compiler':row['compiler'], 'fingerprint':row['fingerprint']}) + if len(self.bodies) > MAX_BODIES: + self.bodies.popitem(last=False) + return result diff --git a/src/decompnet/app/reuse_traversal.py b/src/decompnet/app/reuse_traversal.py new file mode 100644 index 00000000..39902aa0 --- /dev/null +++ b/src/decompnet/app/reuse_traversal.py @@ -0,0 +1,27 @@ +"""Resume cheap candidate discovery fairly, even when cached work consumes the time budget.""" +from contextlib import contextmanager + + +@contextmanager +def rotating(net, project_id, rows, ready=None): + """Persist the last visited function once per pass, including cancellation and early returns.""" + key = f'free-candidate-cursor:{project_id}' + saved = net.store.one('SELECT value FROM settings WHERE key=?', key) + cursor = int(saved['value']) if saved else None + rows = list(rows) + position = next((i+1 for i, row in enumerate(rows) if row['id'] == cursor), 0) + ordered = rows[position:] + rows[:position] + last = None + def visit(): + nonlocal last + for row in ordered: + if ready is not None and not ready(): + return + last = row['id'] + yield row + try: + yield visit() + finally: + if last is not None: + with net.store.tx() as db: + db.execute('INSERT OR REPLACE INTO settings(key,value) VALUES(?,?)', (key, str(last))) diff --git a/src/decompnet/app/similar_matches.py b/src/decompnet/app/similar_matches.py index a94e4911..ded896d4 100644 --- a/src/decompnet/app/similar_matches.py +++ b/src/decompnet/app/similar_matches.py @@ -290,7 +290,8 @@ def run(twins, slug: str, job) -> int: symbol_state = [(r['name'], r['address']) for r in net.store.all( 'SELECT name,address FROM symbols WHERE project_id=? ORDER BY name', proj['id'])] source_cache, ops_cache = {}, {} - from . import address_repairs, regional_layouts + from . import address_repairs, regional_layouts, reuse_donors + donors = reuse_donors.Donors(net, proj) layouts = {pid: regional_layouts.learn(net, proj, other) for pid, other in others.items()} maps = {pid: regional_names(twins, proj, other) for pid, other in others.items()} saved = net.store.one('SELECT function_id FROM reuse_cursors WHERE project_id=?', proj['id']) @@ -312,6 +313,9 @@ def attempt(a, b, other, score, candidate, settings, kind, names, header): current = net.store.one('SELECT status FROM functions WHERE id=?', b['id']) if not current or current['status'] != 'open': return 'skip', '' + from .repair_preflight import invalid_call + if invalid_call(candidate, b['name'], header): + return 'skip', 'Generated call disagrees with a fixed prototype' signature = trial_key(candidate, header, settings, b, [names, symbol_state]) previous = net.store.one("SELECT * FROM reuse_trials WHERE function_id=? AND signature=? AND outcome!='match'", b['id'], signature) @@ -372,14 +376,8 @@ def transfers_exhausted(): continue if net.store.one('SELECT 1 FROM claims WHERE function_id=?', b['id']): continue - rows = net.store.all( - 'SELECT f.*,i.compiler,i.fingerprint FROM cross_game_index i ' - 'JOIN functions f ON f.id=i.function_id AND f.status=\'matched\' ' - 'JOIN projects p ON p.id=i.project_id WHERE i.project_id!=? AND p.arch=? AND p.big_endian=? ' - 'AND i.size BETWEEN ? AND ? AND (i.fingerprint!=? OR i.compiler!=?) ' - 'ORDER BY ABS(i.size-?),i.function_id LIMIT ?', - proj['id'], proj['arch'], proj['big_endian'], - max(MIN_BYTES, int(pb.size * .9)), min(MAX_BYTES, int(pb.size / .9)), pb.key, twins._compiler_key(compiler_for(cfg, b['vram'])), pb.size, MAX_POOL) + rows = donors.candidates(pb.size, max(MIN_BYTES, int(pb.size * .9)), min(MAX_BYTES, int(pb.size / .9)), + pb.key, twins._compiler_key(compiler_for(cfg, b['vram'])), MAX_POOL) layout = [(o, t) for o, t, _ in relocations(b)] rows = [a for a in rows if a['project_id'] in others and a['matched_source'] and ([(o, t) for o, t, _ in relocations(a)] == layout or maps[a['project_id']]) diff --git a/src/decompnet/toolchains.py b/src/decompnet/toolchains.py index 918ef50b..4ed7d8b0 100644 --- a/src/decompnet/toolchains.py +++ b/src/decompnet/toolchains.py @@ -136,6 +136,13 @@ def compiler_family(tid: str) -> str: return _registry_data().get(tid, {}).get("family", "") +def diagnostic_flags(tid: str) -> tuple[str, ...]: + """Warning switches for known GCC backends only; no tools are installed to answer this.""" + spec = _registry_data().get(tid, {}) + gcc = spec.get('family') == 'gcc' or 'gcc' in tid.split('-') + return ('-Wall', '-Wuninitialized') if gcc and spec else () + + def data_dir() -> Path: """Per-user data directory for decompnet (toolchains, caches).""" if sys.platform == "win32": diff --git a/tests/test_compiler_preflight.py b/tests/test_compiler_preflight.py new file mode 100644 index 00000000..59ff490d --- /dev/null +++ b/tests/test_compiler_preflight.py @@ -0,0 +1,50 @@ +"""Warning diagnostics never supply a judge score or change configured flags.""" +import json +from types import SimpleNamespace + +from decompnet import toolchains +from decompnet.app import compiler_preflight + + +def test_diagnostics_are_advisory_cached_and_invalidated_by_actual_inputs(monkeypatch): + monkeypatch.setattr(toolchains, 'diagnostic_flags', lambda tid: ('-Wall', '-Wuninitialized')) + calls = [] + header = ['typedef int word;'] + settings = {'toolchain':'known', 'flags':['-O2']} + proj = {'id':1, 'slug':'game', 'root':'/scratch', 'config':json.dumps({'compiler':settings})} + fn = {'best_source':'int f(void) { int *p; return *p; }', 'vram':16, 'unit':''} + def compile(project, context, source, function): + calls.append(json.loads(project['config'])) + return SimpleNamespace(ok=True, log="func.c:1: warning: 'p' might be used uninitialized", obj=b'unused') + net = SimpleNamespace(cfg=lambda p:json.loads(p['config']), is_rom=lambda p:True, fork_slug=lambda s:None, + _context=lambda *a:{'header':header[0]}, _compile=compile) + assert 'uninitialized' in compiler_preflight.warnings(net, proj, fn) + compiler_preflight.warnings(net, proj, fn) + assert len(calls) == 1 + assert calls[0]['compiler']['flags'] == ['-O2','-Wall','-Wuninitialized'] + assert json.loads(proj['config'])['compiler'] == settings + header[0] += '\nextern int data;' + compiler_preflight.warnings(net, proj, fn) + assert len(calls) == 2 + fn['best_source'] += ' ' + compiler_preflight.warnings(net, proj, fn) + assert len(calls) == 3 + net.fork_slug = lambda s:'fork' + assert not compiler_preflight.warnings(net, proj, fn) + assert len(calls) == 3 + + +def test_warning_flags_do_not_install_unknown_or_non_gcc_tools(): + assert toolchains.diagnostic_flags('gcc-2.7.2-kmc') == ('-Wall', '-Wuninitialized') + assert not toolchains.diagnostic_flags('ido-5.3') + assert not toolchains.diagnostic_flags('unknown') + + +def test_failed_diagnostic_build_cannot_supply_a_warning_or_score(monkeypatch): + monkeypatch.setattr(toolchains, 'diagnostic_flags', lambda tid: ('-Wall',)) + proj = {'id':1,'slug':'game','root':'/scratch','config':json.dumps({'compiler':{'toolchain':'example','flags':[]}})} + net = SimpleNamespace(cfg=lambda p:json.loads(p['config']), is_rom=lambda p:True, fork_slug=lambda s:None, + _context=lambda *a:{'header':''}, + _compile=lambda *a:SimpleNamespace(ok=False, log='warning: not supported', obj=None)) + fn = {'best_source':'int f(void){return 0;}','unit':'','vram':0} + assert compiler_preflight.warnings(net, proj, fn) == '' diff --git a/tests/test_deep_permuter.py b/tests/test_deep_permuter.py index 02565a92..6f97960c 100644 --- a/tests/test_deep_permuter.py +++ b/tests/test_deep_permuter.py @@ -21,6 +21,7 @@ def progress(self, value): def setup(monkeypatch, imported=False): net = FakeNet() + monkeypatch.setattr('decompnet.app.compiler_preflight.warnings', lambda *args: '') net._function = lambda name, slug: net.function() net.local_agent = lambda name: (AGENT, None) net.fork_slug = lambda slug: 'fork' if imported else None @@ -157,3 +158,14 @@ def baseline(self, agent, proj, fn, state, fork): search._score(AGENT, proj, fn, 'cached', 40, None) search._baseline(AGENT, proj, fn, None, None) assert len(seen) == 2 + + +def test_invalid_generated_calls_do_not_consume_the_compile_allowance(monkeypatch): + net = setup(monkeypatch) + search = deep_permuter.Search(net, 1, 4, Log()) + calls = [] + monkeypatch.setattr(Permuter, '_score', lambda *args:calls.append(1) or 40) + net._context = lambda *args: {'header':'', 'version':1} + source = 'int send(int a,int b); int f(void) { return send(1); }' + assert search._score(AGENT, net.project_row('game'), net.function(), source, 40, None) is None + assert search.rejected == 1 and search.tested == 0 and not calls diff --git a/tests/test_free_matching.py b/tests/test_free_matching.py index ab733b33..7aa7c6b8 100644 --- a/tests/test_free_matching.py +++ b/tests/test_free_matching.py @@ -277,3 +277,29 @@ def judge(agent,proj,function,source): detail=pair.repair_feedback.detail('destination',case) assert detail['evidence']['provenance']=={'kind':'automatic candidate','family':'template','origin':'MIPS leaf template'} assert detail['status']=='new' and _TRIAL.get() is None + + +def test_cached_discovery_yields_time_and_resumes_after_last_visited_function(pair, monkeypatch): + import threading + + from decompnet.app import free_candidates + clock = [0.0] + monkeypatch.setattr(free_candidates, 'time', SimpleNamespace(monotonic=lambda:clock[0], time=lambda:0)) + target = fn([0x03e00008,0x24020007])['target'] + with pair.store.tx() as db: + db.execute('UPDATE functions SET target=?,size=8,n_insns=2 WHERE id=2', (target,)) + db.execute("INSERT INTO functions(id,project_id,name,target,size,n_insns,status,vram,relocs) " + "SELECT 3,project_id,'next',target,size,n_insns,'open',vram+16,relocs FROM functions WHERE id=2") + visits = [] + def cached_templates(function, *args): + visits.append(function['id']) + clock[0] += 6 # Discovery, not compilation, consumes the phase's share of ten seconds. + return [] + monkeypatch.setattr(free_candidates.mips_templates, 'candidates', cached_templates) + for _ in range(2): + job = Job(stop=threading.Event()) + free_candidates.budget(job, seconds=10, attempts=8) + free_candidates.run(pair.twins, 'destination', job) + assert job.reuse_attempts == 0 + assert clock[0] < job.reuse_deadline # Wider discovery still gets time. + assert visits == [2, 3] diff --git a/tests/test_local_experiments.py b/tests/test_local_experiments.py index e8c80009..d4f46bdd 100644 --- a/tests/test_local_experiments.py +++ b/tests/test_local_experiments.py @@ -23,7 +23,7 @@ def compile(agent,function,text,slug): _,rev=agent_tokens.candidate(net,a,'game','f') args={'function':'f','revision':rev,'families':['signedness','declaration_order'],'max_trials':2} result=tools.call(net,a,'game','try_local_repairs',args) - assert len(seen)==2 and '1: signedness, 2: declaration_order' in result + assert len(seen)==2 and '1: signedness, 2: declaration_order' in result, result assert 'starting candidate retained' in result assert agent_tokens.candidate(net,a,'game','f')[0]==source # Submitted local variants are remembered; a second call advances instead of replaying them. diff --git a/tests/test_repair_preflight.py b/tests/test_repair_preflight.py new file mode 100644 index 00000000..ef74648f --- /dev/null +++ b/tests/test_repair_preflight.py @@ -0,0 +1,36 @@ +"""Preflight skips only fixed, plain C arity errors, never guesses about ambiguous C.""" +import pytest + +from decompnet.app.repair_preflight import invalid_call + + +def test_known_prototype_rejects_generated_missing_argument(): + source = 'extern int send(int a, int b); int f(int x) { return send(x); }' + assert 'expects 2' in invalid_call(source, 'f') + assert not invalid_call(source.replace('send(x);', 'send(x, 7);'), 'f') + assert invalid_call('int f(void) { return send(); }', 'f', 'extern int send(int x);') + + +@pytest.mark.parametrize('prototype,call', [ + ('int send();', 'send()'), ('int send(int x); int send(int x,...);', 'send(1,2)'), + ('#include \"library.h\"\nint send(int x);', 'send()'), ('int send(int x, ...);', 'send(1, 2, 3)'), + ('int send(word x);', 'send()'), ('int send(int x=1);', 'send()'), + ('int send(int x); int send(int x, int y);', 'send()'), + ('#define X 1,2\nint send(int x,int y);', 'send(X)'), + ('#if FLAG\nint send(int x);\n#endif', 'send()'), +]) +def test_ambiguous_declarations_stay_with_compiler(prototype, call): + assert not invalid_call(prototype+'\nint f(void) { return '+call+'; }', 'f') + + +def test_nested_calls_strings_comments_and_void(): + header = 'void send(int x, char *s); void finish(void);' + assert not invalid_call('int f(void) { send(other(1, 2), "a,b"); finish(); /* send() */ }', 'f', header) + assert invalid_call('int f(void) { finish(1); }', 'f', header) + assert not invalid_call('int f(void) { int (*send)(); return send(); }', 'f', header) + + +def test_draft_generation_removes_invalid_arity_before_bounded_selection(): + from decompnet.app.draft_repairs import variants + source = 'int send(int x,int y); int f(int x) { return send(x, 7); }' + assert all('return send(x);' not in candidate for candidate in variants(source, 'f', 'mips')) diff --git a/tests/test_reuse_donors.py b/tests/test_reuse_donors.py new file mode 100644 index 00000000..4623523d --- /dev/null +++ b/tests/test_reuse_donors.py @@ -0,0 +1,29 @@ +"""A pass-local donor cache keeps the old SQL shortlist semantics without repeated source reads.""" +import pytest + +from decompnet.app.reuse_donors import Donors +from tests.test_similar_matches import pair as pair_fixture + +pair = pytest.fixture(pair_fixture.__wrapped__) + + +def test_shortlist_keeps_order_limits_and_reloads_on_a_new_pass(pair, monkeypatch): + proj = pair._project('destination') + pair.twins.index_project(pair._project('source')) + donors = Donors(pair, proj) + original = pair.store.one + reads = [] + def one(sql, *args): + if sql.startswith('SELECT * FROM functions WHERE id='): + reads.append(args) + return original(sql, *args) + monkeypatch.setattr(pair.store, 'one', one) + first = donors.candidates(32, 0, 3200, '', '', 400) + assert first and first[0]['id'] == 1 + assert donors.candidates(32, 0, 3200, '', '', 400) == first + assert len(reads) == 1 + assert not donors.candidates(32, 0, 3200, first[0]['fingerprint'], first[0]['compiler'], 400) + assert not donors.candidates(32, 0, 3200, '', '', 0) + with pair.store.tx() as db: + db.execute("UPDATE functions SET status='open' WHERE id=1") + assert not Donors(pair, proj).candidates(32, 0, 3200, '', '', 400) diff --git a/tests/test_reuse_traversal.py b/tests/test_reuse_traversal.py new file mode 100644 index 00000000..23d40e18 --- /dev/null +++ b/tests/test_reuse_traversal.py @@ -0,0 +1,23 @@ +"""Discovery continues past cached prefixes without mixing the wider similarity cursor.""" +from decompnet.app.reuse_traversal import rotating +from tests.test_permute import FakeNet + + +def test_cursor_survives_early_return_and_wraps_in_size_order(): + net = FakeNet() + rows = [{'id':9}, {'id':2}, {'id':5}] + with rotating(net, 1, rows) as functions: + assert next(functions)['id'] == 9 + with rotating(net, 1, rows) as functions: + assert [r['id'] for r in functions] == [2,5,9] + # A removed/matched cursor restarts safely instead of skipping new work. + with rotating(net, 1, rows[1:]) as functions: + assert next(functions)['id'] == 2 + try: + with rotating(net, 1, rows) as functions: + assert next(functions)['id'] == 5 + raise RuntimeError('cancelled') + except RuntimeError: + pass + with rotating(net, 1, rows) as functions: + assert next(functions)['id'] == 9 From 3e39bcbb5dcbc94bba6c1d1ae1e089ed8c3e2b01 Mon Sep 17 00:00:00 2001 From: RetroGameDeveloper <40120498+RetroGameDeveloper@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:35:29 +0000 Subject: [PATCH 2/2] Guard shared diagnostic cache publication across agents --- src/decompnet/app/compiler_preflight.py | 30 +++++++++++++++---------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/decompnet/app/compiler_preflight.py b/src/decompnet/app/compiler_preflight.py index c76a606d..16e522f2 100644 --- a/src/decompnet/app/compiler_preflight.py +++ b/src/decompnet/app/compiler_preflight.py @@ -6,6 +6,7 @@ import hashlib import json import logging +import threading from .. import toolchains from ..project import compiler_for @@ -24,17 +25,22 @@ def warnings(net, proj, fn): return '' header = net._context(proj['id'], fn['unit'])['header'] key = hashlib.sha256(repr((proj['root'], settings, header, fn['best_source'])).encode()).hexdigest() - cache = getattr(net, '_compiler_preflights', {}) - net._compiler_preflights = cache - if key not in cache: - diagnostic = {**cfg, 'compiler_ranges': [], 'compiler': {**settings, 'flags': [*settings.get('flags', []), *flags]}} - try: - result = net._compile({**dict(proj), 'config': json.dumps(diagnostic)}, header, fn['best_source'], fn) - lines = [line for line in result.log.splitlines() if 'warning' in line.lower() or 'uninitialized' in line.lower()] - cache[key] = '\n'.join(lines)[:1000] if result.ok else '' - except Exception: # advisory evidence must never stand between the user and the real judge - log.exception('compiler warning preflight failed') - return '' # Tool failures remain retryable. + cache = net.__dict__.setdefault('_compiler_preflights', {}) + lock = net.__dict__.setdefault('_compiler_preflight_lock', threading.Lock()) + with lock: + if key in cache: + return cache[key] + diagnostic = {**cfg, 'compiler_ranges': [], 'compiler': {**settings, 'flags': [*settings.get('flags', []), *flags]}} + try: + result = net._compile({**dict(proj), 'config': json.dumps(diagnostic)}, header, fn['best_source'], fn) + lines = [line for line in result.log.splitlines() if 'warning' in line.lower() or 'uninitialized' in line.lower()] + text = '\n'.join(lines)[:1000] if result.ok else '' + except Exception: # advisory evidence must never stand between the user and the real judge + log.exception('compiler warning preflight failed') + return '' # Tool failures remain retryable. + # Different functions may compile concurrently. Only publication/eviction needs the lock. + with lock: + cache[key] = text while len(cache) > 256: del cache[next(iter(cache))] - return cache[key] + return text