Feed random search wins into local repairs and recover blocked deep passes

README.md

@@ -44,6 +44,14 @@ project-wide prototypes stay unchanged. These are hypotheses: an improved simila
 and only a byte-exact result counts as matched. The families also reach iterative/deep search and
 `try_local_repairs`; loops, incoming jumps, shadowing and ambiguous declarations limit what is proposed.
 
+Deep search now revisits local repairs after a random rewrite improves the saved candidate. Each burst
+tries at most 32 proposals and shares the original time/compile allowance. Small constant/copy reuse and
+final integer shift/scale candidates target register-only differences without assigning machine registers.
+A score-zero open function is still freshly verified before being marked matched. If an otherwise unusable
+ROM draft conflicts with one plain shared return declaration, deep search can align the candidate to that
+header and remeasure it. The old draft stays in history, the shared header stays unchanged, and the new
+baseline may have a lower percentage. Ambiguous signatures still need review.
+
 For close ROM misses it also tests cached compiler flag alternatives, two probes per pass. Later passes skip
 already tested choices and can reach optimization levels as well as debug flags. Only exact results qualify for a
 narrow compiler-range change. When automatic compiler changes are disabled, the result becomes a suggestion for

ROADMAP.md

@@ -1,5 +1,10 @@
 # Roadmap
 
+Deep search now chains bounded local repairs after random improvements. Register reuse hypotheses share
+all normal draft/permuter paths. Fresh exact judgments repair score-zero/open bookkeeping; plain return
+conflicts can recover a buildable ROM candidate with an honest baseline and retained history. Adaptive
+per-function budgets and a pool of alternative search candidates remain planned follow-ups.
+
 Bounded integer/call repairs now include evidence-led local widening, candidate-owned callee prototype widths,
 integer lifetime splits/merges, call distribution into branches, terminal goto leaves and switch layout variants.
 They share the ROM/imported-project judge and existing budgets; a better similarity score remains a draft.

src/decompnet/app/deep_permuter.py

@@ -10,9 +10,12 @@
 from .. import scoring
 from ..network import NetworkError
 from ..project import compiler_for
+from ..repair_feedback import trial as repair_trial
 from . import agent_tokens, draft_repairs, permuter_local, stack_padding_repairs
 from .permute import Permuter, State
 
+AFTER_RANDOM_TRIALS = 32
+
 
 class SearchLimit(Exception):
     pass
@@ -51,6 +54,10 @@ def __init__(self, net, minutes, max_variants, log, fn=None):
         self.measuring_baseline = False
         self.record_improvement = lambda score: None
         self.recorded_best = None
+        self.local_seen = set()
+        self.refinement_rounds = 0
+        self.winner_diff = None
+        self.winner_source = None
 
     def check(self):
         self.log.check_cancel()
@@ -74,17 +81,99 @@ def report(self):
             self.recorded_best = self.best
         self.log.progress({'phase': self.phase, 'tested': self.tested, 'skipped': self.skipped,
                            **self.metrics(), 'variant_limit': self.limit, 'rejected': self.rejected,
-                           'seconds_left': max(0, round(self.deadline-time.monotonic()))})
+                           'seconds_left': max(0, round(self.deadline-time.monotonic())), 'refinement_rounds': self.refinement_rounds})
 
-    def _baseline(self, *args):
-        # A cached scalar score cannot supply the instruction rows needed by targeted repairs.
-        # The parent still caches a measured baseline within this search instance.
+    def _baseline(self, agent, proj, fn, state, fork):
+        # Baseline evidence must come from today's compiler/header, including score-zero open rows.
         self.measuring_baseline = True
         try:
-            return super()._baseline(*args)
+            measured = super()._baseline(agent, proj, fn, state, fork)
+            if measured[0] is not None or fork:
+                return measured
+            from .prototype_alignment import variants
+            header = self.net._context(proj['id'], fn['unit'])['header']
+            for candidate in variants(fn['best_source'], fn['name'], header):
+                score = self._score(agent, proj, fn, candidate, float('inf'), fork)
+                fresh = self.net._function(fn['name'], proj['slug'])
+                if score is not None and fresh['status'] == 'open' and fresh['best_source'] == fn['best_source']:
+                    # The old percentage described C which no longer compiles. Retain that source in
+                    # history and rebase only this broken draft after a normal compiler/judge result.
+                    self.net._record(fn, agent, None, fn['best_source'],
+                                     self.net._context(proj['id'], fn['unit'])['version'],
+                                     compiled=False, via='permuter')
+                    with self.net.store.tx() as db:
+                        current = self.net._function(fn['name'], proj['slug'])
+                        unchanged = (current['status'] == 'open' and current['best_source'] == fn['best_source']
+                                     and all(current[k] == fn[k] for k in ('best_score', 'target', 'relocs', 'vram'))
+                                     and self.net._project(proj['slug'])['config'] == proj['config']
+                                     and self.net._context(proj['id'], fn['unit'])['header'] == header)
+                        if unchanged:
+                            db.execute("UPDATE functions SET best_source=?,best_score=?,best_agent_id=? WHERE id=? AND status='open'",
+                                       (candidate, score, agent.id, fn['id']))
+                    fresh = self.net._function(fn['name'], proj['slug'])
+                if score is not None and (fresh['best_source'] == candidate or fresh['status'] == 'matched'):
+                    self.log('Repaired a conflicting return type in the candidate; shared header unchanged. '
+                             'The broken draft is retained in history; the new baseline may have a lower percentage.')
+                    return score, measured[1]+1
+            return measured
         finally:
             self.measuring_baseline = False
 
+    def local_repairs(self, agent, proj, fn, best, fork, max_trials=None):
+        """Regenerate from each saved winner, sharing this search's deadline, cache and allowance."""
+        flags = compiler_for(self.net.cfg(proj), fn['vram']).get('flags', []) if not fork else []
+        trials = 0
+        source = fn['best_source']
+        while best > 0:
+            self.check()
+            self.local_seen.add(source)
+            rows = getattr(self.net, '_last_diffs', {}).get((agent.id, fn['id']), ([], ''))[0]
+            labels = {}
+            choices = draft_repairs.variants(source, fn['name'], proj['arch'], fn['asm'] or '', flags, labels=labels, limit=None)
+            choices += stack_padding_repairs.grow(source, fn['name'], proj['arch'], rows)
+            choices += [s for _, s in permuter_local.variants(source, fn['name'])]
+            for candidate in dict.fromkeys(choices):
+                if candidate in self.local_seen:
+                    continue
+                if max_trials is not None and trials >= max_trials:
+                    return best, source
+                self.local_seen.add(candidate)
+                trials += 1
+                with repair_trial({'kind': 'deep local repair', 'family': labels.get(candidate, 'local_repair'),
+                                   'base_source_sha256': hashlib.sha256(source.encode()).hexdigest()}):
+                    score = self._score(agent, proj, fn, candidate, best, fork)
+                if score is not None and score < best:
+                    best = score
+                    fn = self.net._function(fn['name'], proj['slug'])
+                    source = fn['best_source']
+                    break
+            else:
+                break
+        return best, source
+
+    def _after_improvement(self, agent, proj, fn, state, fork, best, source):
+        previous = self.phase
+        self.phase = 'local repairs after random improvement'
+        self.refinement_rounds += 1
+        try:
+            self.log('Random rewrite improved the candidate; trying targeted local repairs on its new diff.')
+            # Keep the exploratory lane alive: at most 32 local proposals after each random win.
+            # The original global time/compile limit always wins over this burst limit.
+            fresh = self.net._function(fn['name'], proj['slug'])
+            return self.local_repairs(agent, proj, fresh, best, fork, max_trials=AFTER_RANDOM_TRIALS)
+        finally:
+            if self.winner_diff is not None:
+                # A losing local trial must not steer the next random batch's weights. Restore
+                # evidence for the saved winner, without spending another compiler invocation.
+                if hasattr(self.net, 'keep_diff'):
+                    self.net.keep_diff(agent.id, fn['id'], self.winner_diff[0])
+                else:
+                    self.net._last_diffs[agent.id, fn['id']] = self.winner_diff
+                if hasattr(self.net, '_agent_outcomes'):
+                    self.net._agent_outcomes[agent.id, fn['id']] = {
+                        'source': self.winner_source, 'score': self.best, 'rows': self.winner_diff[0]}
+            self.phase = previous
+
     def _score(self, agent, proj, fn, source, best, fork):
         self.check()
         self.visited += 1
@@ -112,13 +201,19 @@ def _score(self, agent, proj, fn, source, best, fork):
                     db.execute('INSERT OR REPLACE INTO reuse_trials(function_id,signature,outcome,reason,feedback) VALUES(?,?,?,?,?)',
                                (fn['id'], key, 'miss', 'deep function search', json.dumps(score)))
         if score is not None:
+            winner = False
             if self.measuring_baseline and not self.baseline_measured:
                 self.starting_score = self.best = score
                 self.baseline_measured = True
+                winner = True
             elif not reused and not self.measuring_baseline and (self.best is None or score < self.best):
                 if self.best is not None:
                     self.improvements += 1
                 self.best = score
+                winner = True
+            if winner:
+                self.winner_diff = getattr(self.net, '_last_diffs', {}).get((agent.id, fn['id']))
+                self.winner_source = source
         self.report()
         return score
 
@@ -147,39 +242,16 @@ def run(net, slug, name, minutes, max_variants, log):
             raise NetworkError('This function was claimed meanwhile; try again after it is released.')
         try:
             fork = net.fork_slug(slug)
-            flags = compiler_for(net.cfg(proj), fn['vram']).get('flags', []) if not fork else []
-            # Fresh baseline evidence enables measured stack repairs and diagnoses stale saved scores.
+            # Fresh baseline evidence also diagnoses obsolete candidate-owned prototypes.
             baseline, _ = search._baseline(agent, proj, fn, state, fork)
             if baseline is None:
-                raise NetworkError('The saved C does not compile in the current context; fix it before deep search.')
+                raise NetworkError('The saved C does not compile in the current context; fix its declarations before deep search.')
             if baseline == 0:
                 reason = 'Matched'
             else:
-                best = baseline
-                seen = {fn['best_source']}
-                repair_fn = fn
-                while best > 0:
-                    search.check()
-                    source = repair_fn['best_source']
-                    rows = getattr(net, '_last_diffs', {}).get((agent.id, repair_fn['id']), ([], ''))[0]
-                    choices = draft_repairs.variants(source, repair_fn['name'], proj['arch'], repair_fn['asm'] or '', flags, limit=None)
-                    choices += stack_padding_repairs.grow(source, repair_fn['name'], proj['arch'], rows)
-                    choices += [s for _,s in permuter_local.variants(source, repair_fn['name'])]
-                    for candidate in dict.fromkeys(choices):
-                        if candidate in seen:
-                            continue
-                        seen.add(candidate)
-                        score = search._score(agent, proj, repair_fn, candidate, best, fork)
-                        if score == 0:
-                            reason = 'Matched'
-                        if score is not None and score < best:
-                            best = score
-                            # Chain repairs from the saved winner and its new diff; keep the same
-                            # deadline/compile budget and avoid retesting candidates from earlier rounds.
-                            repair_fn = net._function(name, slug)
-                            break
-                    else:
-                        break
+                best, _ = search.local_repairs(agent, proj, net._function(name, slug), baseline, fork)
+                if best == 0:
+                    reason = 'Matched'
         finally:
             search._let_go(agent, fn)
         if reason != 'Matched':
@@ -203,7 +275,8 @@ def run(net, slug, name, minutes, max_variants, log):
     search.report()
     log(f'{reason}: {search.tested} candidates tested, {search.skipped} cached misses skipped, {search.rejected} invalid calls skipped. Verified improvements are saved immediately.')
     result = {'function': name, 'reason': reason, 'tested': search.tested, 'skipped': search.skipped,
-              'rejected': search.rejected, 'matched': net._function(name, slug)['status'] == 'matched', **search.metrics()}
+              'rejected': search.rejected, 'refinement_rounds': search.refinement_rounds,
+              'matched': net._function(name, slug)['status'] == 'matched', **search.metrics()}
     if sweeps:
         sweeps.completed(slug, fn, minutes, max_variants, result, build_scope)
     return result

src/decompnet/app/deep_sweep.py

@@ -16,8 +16,17 @@
 
 
 def signature(net, proj, fn, minutes, variants):
-    return hashlib.sha256(repr(('deep-pass-v1', agent_tokens.target_scope(net, fn, ''),
-                               fn['best_source'], minutes, variants)).encode()).hexdigest()
+    inputs = ('deep-pass-v1', agent_tokens.target_scope(net, fn, ''), fn['best_source'], minutes, variants)
+    # Retry the two formerly blocked states once under the new recovery rules. Keep unrelated
+    # completed passes resumable; users can explicitly repeat them to try the new search recipes.
+    if fn['status'] == 'open' and fn['best_score'] == 0:
+        inputs += ('reverify-zero-v1',)
+    elif fn['best_source'] and not net.fork_slug(proj['slug']):
+        from .prototype_alignment import variants as align
+        ctx = net._context(proj['id'], fn['unit'])
+        if ctx and align(fn['best_source'], fn['name'], ctx['header']):
+            inputs += ('return-alignment-v1',)
+    return hashlib.sha256(repr(inputs).encode()).hexdigest()
 
 
 class Sweeps:

src/decompnet/app/draft_repairs.py

@@ -24,6 +24,7 @@
     pattern_repairs,
     pointer_cast_repairs,
     register_repairs,
+    register_reuse_repairs,
     stack_padding_repairs,
     stack_repairs,
     storage_scope_repairs,
@@ -57,6 +58,9 @@ def variants(source: str, name: str, arch: str, asm: str = '', flags=(), labels=
     for family, text in register_repairs.variants(source, name, arch):
         proposals.append(text)
         labels[text] = family
+    for family, text in register_reuse_repairs.variants(source, name):
+        proposals.append(text)
+        labels[text] = family
     for text in mmio_repairs.variants(source, name, arch):
         proposals.append(text)
         labels[text] = 'mmio_address'

src/decompnet/app/local_experiments.py

@@ -21,12 +21,13 @@
     pattern_repairs,
     pointer_cast_repairs,
     register_repairs,
+    register_reuse_repairs,
     repair_ranking,
     stack_padding_repairs,
     store_address_repairs,
 )
 
-FAMILIES = ('signedness','declaration_order','typed_temporary','branch_orientation','narrowing','source_lines',
+FAMILIES = ('constant_register_reuse', 'copy_register_reuse', 'last_shift_reuse', 'signedness','declaration_order','typed_temporary','branch_orientation','narrowing','source_lines',
             'address_temporary','constant_local','comparison_load','stack_array','loop_induction',
             'parameter_width','value_lifetime','sum_shape','call_result_inline','comparison_reuse','integer_address_order','mmio_address','call_arity','struct_byte_offset','switch_call','paired_load_order','array_index_width','split_scalar_lifetime','pointer_before_store','store_address_lifetime','stack_padding','local_width','pointer_cast_inline',
             'local_widening','callee_parameter_width','callee_return_width','split_integer_lifetime','merge_integer_lifetimes','branch_call','goto_leaf','switch_layout')
@@ -69,7 +70,7 @@ def run(net, agent, slug, function, revision, families=None, max_trials=6):
         for family,candidate in lifetime_repairs.variants(source,fn['name']):
             if family in pools:
                 pools[family].append(candidate)
-        for module in (callee_prototype_repairs, integer_lifetime_repairs, branch_layout_repairs):
+        for module in (callee_prototype_repairs, integer_lifetime_repairs, branch_layout_repairs, register_reuse_repairs):
             for family, candidate in module.variants(source, fn['name']):
                 if family in pools:
                     pools[family].append(candidate)

src/decompnet/app/permute.py

@@ -365,6 +365,10 @@ def _let_go(self, agent, fn) -> None:
         with self.net.store.tx() as db:
             db.execute("DELETE FROM claims WHERE function_id=? AND agent_id=?", (fn["id"], agent.id))
 
+    def _after_improvement(self, agent, proj, fn, st, fork, best, source):
+        """Deep search can run bounded local repairs here without changing normal random turns."""
+        return best, source
+
     def _permute(self, agent, proj, fn, st: State) -> None:
         seconds = self.seconds
         if st.stop.is_set() or not self._hold(agent, fn, seconds=seconds):
@@ -435,13 +439,18 @@ def _permute(self, agent, proj, fn, st: State) -> None:
                         continue
                     if score < best:
                         best, best_src = score, cand
+                        st.add(improved=1)
+                        if best > 0:
+                            best, best_src = self._after_improvement(agent, proj, fn, st, fork, best, best_src)
                         rows = repair_ranking.evidence(self.net, fn['id'], agent.id)
                         focus, boosts = permuter_focus.classify(rows), permuter_focus.boosts(rows)
-                        st.add(improved=1)
-                        if score == 0:
+                        if best == 0:
                             st.add(matched=1)
                             log.info("permuter: %s matched", fn["name"])
                             break
+                        if best < score:
+                            cur, cur_src = best, best_src
+                            break  # generate the next batch from the refined winner, not its parent
                     if score <= cur or rng.random() < 0.02:  # mostly downhill, now and then sideways off a plateau
                         cur, cur_src = score, cand
         finally:
@@ -574,6 +583,6 @@ def _score(self, agent, proj, fn, source: str, best: int, fork: str | None) -> i
         if not hasattr(self.net, '_agent_outcomes'):
             self.net._agent_outcomes = {}
         self.net._agent_outcomes[agent.id, fn['id']] = {'source': source, 'score': d.score, 'rows': d.rows}
-        if d.score < best:
+        if d.score < best or (d.bytes_match and fn['status'] != 'matched'):
             self.net._record(fn, agent, d.score, source, ctx["version"], matched=d.bytes_match, via="permuter", effective_settings=getattr(result,"effective_settings",None))
         return 0 if d.bytes_match else max(1, d.score)