Speed up ranked reclamation with ordered retired-register buckets

certificates/joint-dual-kappa.json

@@ -838,11 +838,11 @@
     "scripts/experiments/binary_frame_replay.py": "392e78e62d7685c99a5888a7da743ea6c3dcad631af4a743c91e0c09348442bc",
     "scripts/experiments/joint_dual_compiler.py": "853d3fc4a740c5b74c23b3a6af5d4dc9b41fb38e9b55678539a7e0ee71947c13",
     "scripts/experiments/joint_dual_compose.py": "741900aa6f69c0e3458700f3b534334269004ad80c50405ecfeda18e6d3f2800",
-    "scripts/experiments/joint_dual_reclaim_compiler.py": "9984aaccf880e4840402322d76678bfddb33ce7e494af78430a43429f51b85fd",
+    "scripts/experiments/joint_dual_reclaim_compiler.py": "f0696953d057239ed5a75e28d857b814d43ca43deaaf7b7b8fd191aa115112ff",
     "scripts/experiments/pin_joint_dual_sources.py": "4d31cc0e38b7902709ca5f8f714cba109760d2592346dd7f8dc0b900a2d9fe00",
     "scripts/experiments/verify_joint_dual.py": "7a31bba1e9c0ded32ae285df2ca9bc91ef20c330f6789fdba361876f3c7a1985"
   },
-  "source_manifest_sha256": "3d8a3907630068fb76c0ed3a3e8d33245265471d1770d262b489876a65493155",
+  "source_manifest_sha256": "c8649e811e43b33a7281746fd61e9adbd1370690ef7c2ec114a6f2d61016704f",
   "sources": [
     {
       "files": {

certificates/joint-dual-validation.json

@@ -226,7 +226,7 @@
     }
   },
   "bit_saving": "2382370177/50000000000000",
-  "certificate_sha256": "16f18ad60af972ce1f9fbbbb1277a69ad838727d542889bbac6c5539d5420bf7",
+  "certificate_sha256": "4044f15abf49b1551193891836e2d20975b8246acc888180af47266c64f77af0",
   "kappa": "4764513337/100000000000000",
   "margins": 7,
   "producer_words_regenerated_and_matched": true,

research/joint-dual/SOURCE.json

@@ -50,7 +50,7 @@
     "scripts/experiments/binary_frame_replay.py": "392e78e62d7685c99a5888a7da743ea6c3dcad631af4a743c91e0c09348442bc",
     "scripts/experiments/joint_dual_compiler.py": "853d3fc4a740c5b74c23b3a6af5d4dc9b41fb38e9b55678539a7e0ee71947c13",
     "scripts/experiments/joint_dual_compose.py": "741900aa6f69c0e3458700f3b534334269004ad80c50405ecfeda18e6d3f2800",
-    "scripts/experiments/joint_dual_reclaim_compiler.py": "9984aaccf880e4840402322d76678bfddb33ce7e494af78430a43429f51b85fd",
+    "scripts/experiments/joint_dual_reclaim_compiler.py": "f0696953d057239ed5a75e28d857b814d43ca43deaaf7b7b8fd191aa115112ff",
     "scripts/experiments/pin_joint_dual_sources.py": "4d31cc0e38b7902709ca5f8f714cba109760d2592346dd7f8dc0b900a2d9fe00",
     "scripts/experiments/verify_joint_dual.py": "7a31bba1e9c0ded32ae285df2ca9bc91ef20c330f6789fdba361876f3c7a1985",
     "tests/test_joint_reclaim.py": "e2cfc99a52283c4912f970f83c7da5481ba9840d5f999f7e0e5468e8e8862a41"

scripts/experiments/joint_dual_reclaim_compiler.py

@@ -18,6 +18,7 @@
 from changed_graph import graph
 from collections import defaultdict,Counter,deque
 import argparse,json,time,struct
+from bisect import bisect_left, insort
 
 def independent(rows,vec):
  for row in rows:
@@ -117,10 +118,26 @@ def compile_(h,matching=True,reclaim=False,dirty=True):
  edges,chosen,right,stats=match(blocks,uses,matching)
  print('matched',len(chosen),'seconds',time.time()-t0,flush=True,file=sys.stderr)
  slots=[];frames=[];ops=[];events=[];hist=Counter();assign={};sources={};retired=set();stats['regions']=len(blocks);stats['matched']=len(chosen);stats['multi_node_regions']=sum(len(b['nodes'])>1 for b in blocks)
+ # Keep the original descending-rank, ascending-slot selection order.
+ # A retired anchor can move during clearing: raise_ updates this index too.
+ retired_by_rank=[[] for _ in range(h+1)]
+ def retire_add(s):
+  assert s not in retired
+  retired.add(s);insort(retired_by_rank[blocks[frames[s]]['rank']],s)
+ def retire_remove(s):
+  bucket=retired_by_rank[blocks[frames[s]]['rank']]
+  i=bisect_left(bucket,s);assert i<len(bucket) and bucket[i]==s
+  bucket.pop(i);retired.remove(s)
  def new(g):
   s=len(slots);slots.append(0);frames.append(g);hist[blocks[g]['rank']]+=1;events.append((s,-1,g));return s
  def raise_(s,g):
-  old=frames[s];assert contains(blocks[old]['frame'],blocks[g]['frame']);hist[blocks[g]['rank']-blocks[old]['rank']]+=1;frames[s]=g;events.append((s,old,g))
+  old=frames[s];assert contains(blocks[old]['frame'],blocks[g]['frame'])
+  oldrank,newrank=blocks[old]['rank'],blocks[g]['rank']
+  if s in retired and oldrank!=newrank:
+   bucket=retired_by_rank[oldrank];i=bisect_left(bucket,s)
+   assert i<len(bucket) and bucket[i]==s
+   bucket.pop(i);insort(retired_by_rank[newrank],s)
+  hist[newrank-oldrank]+=1;frames[s]=g;events.append((s,old,g))
  def xor(a,b,g):
   assert a!=b;raise_(a,g);raise_(b,g);slots[a]^=slots[b];ops.append((a,b,g))
  def acquire(g,anchors):
@@ -134,15 +151,15 @@ def ins(s):
      old,e=bb[p];row^=old;expr^=e
     return expr
    for s in anchors:ins(s)
-   for s in sorted(retired,key=lambda s:(-blocks[frames[s]]['rank'],s)):
+   for s in (s for rank in range(h,-1,-1) for s in retired_by_rank[rank]):
     if not contains(blocks[frames[s]]['frame'],blocks[g]['frame']):continue
     e=ins(s)
     if e is None:continue
     assert e>>s&1;e^=1<<s;aa=[]
     while e:
      bit=e&-e;aa.append(bit.bit_length()-1);e^=bit
     for a in aa:xor(s,a,g)
-    assert not slots[s];retired.remove(s);stats['reclaimed']+=1;stats['clearing_xors']+=len(aa);return s
+    assert not slots[s];retire_remove(s);stats['reclaimed']+=1;stats['clearing_xors']+=len(aa);return s
   return new(g)
  for step,g in enumerate(order):
   b=blocks[g];outuses=[u for u in b['uses']if u not in right];outvalues=sorted({uses[u][0]for u in outuses});assert outvalues==b['outvalues']
@@ -179,7 +196,7 @@ def ins(s):
    for s,(kind,key)in zip(ins,labels):
     if kind=='value':value_slots[key]=s;assert slots[s]==signal[key]
     elif kind=='carry':assign[key]=s;assert slots[s]==signal[uses[key][0]]
-    else:retired.add(s);spare.append(s)
+    else:retire_add(s);spare.append(s)
    # Express every dependent desired output in the new independent basis.
    ob=list(value_slots);bb={}
    for i,x in enumerate(ob):