Research notes (no κ claim): E8 decoder floors are tight; extension-ring, triple and Lorentzian units lose after helpers; cold-end, dispatch and fusion-search negatives

research/e8-decoder-floors-and-cold14-notes/ade_screen.py

@@ -0,0 +1,40 @@
+import sys, itertools, numpy as np
+from fractions import Fraction as Fr
+sys.path.insert(0, '/Users/joelpulikkan/imb-main/research/format-limits/scripts')
+from e8units import f2_rank
+from skel import X
+
+def roots_A(n):  # in R^{n+1}
+    R=[]
+    for i in range(n+1):
+        for j in range(i+1,n+1):
+            v=[0]*(n+1); v[i]=1; v[j]=-1; R.append(v)
+    return np.array(R)
+def roots_D(n):
+    R=[]
+    for i in range(n):
+        for j in range(i+1,n):
+            for s in (1,-1):
+                v=[0]*n; v[i]=1; v[j]=s; R.append(v)
+    return np.array(R)
+def roots_E8():
+    R=[list(r) for r in roots_D(8)]
+    for signs in itertools.product((1,-1),repeat=8):
+        if signs[0]==1 and signs.count(-1)%2==0: R.append([s*0.5 for s in signs])
+    return np.array(R)
+def E7():
+    R=roots_E8(); s=np.ones(8)*0.5
+    return R[np.abs(R@s)<1e-9]
+def E6():
+    R=roots_E8(); a=np.ones(8)*0.5; b=np.zeros(8); b[6]=1;b[7]=1  # pick two orthogonal roots
+    b=R[[i for i in range(len(R)) if abs(R[i]@a)<1e-9][0]]
+    R7=R[np.abs(R@a)<1e-9]; return R7[np.abs(R7@b)<1e-9]
+def stats(name,R):
+    G=np.rint(R@R.T).astype(int); v=int(len(R)); h=int(np.linalg.matrix_rank(G))
+    B=(1+G)%2; c=int(f2_rank(B))
+    x=Fr(c*h, v)
+    vals=[(s,X(s,h,1,x)) for s in (3,5,7,9)]; vals=[(s,float(r["hi"])) for s,r in vals if r]; best=max(vals,key=lambda t:t[1]) if vals else (0,0.0)
+    print(f"{name:6s} v={v:4d} h={h:2d} c={c:3d} x~c*h/v={float(x):.3f}  X*={best[1]:.4e}@s{best[0]}")
+for n in (4,5,6,7,8): stats(f"A{n}",roots_A(n))
+for n in (4,5,6,7,8,9,10,12): stats(f"D{n}",roots_D(n))
+stats("E6",E6()); stats("E7",E7()); stats("E8",roots_E8())

research/e8-decoder-floors-and-cold14-notes/ade_screen_fixed.py

@@ -0,0 +1,131 @@
+"""Corrected ADE unit screen (replaces #620's ade_screen.py; erratum found by #630).
+
+Per unit (real root lines, window h = Gram rank, k = 1):
+  v, h                 exact
+  c0 = rank_F2(J + G)  the label decoder B0 used by #620 (an UPPER bound on the F2 minrank of the pattern)
+  omega                largest pairwise non-orthogonal set (identity block of the pattern) -> minrank >= omega
+  c = minrank          exact F2 minrank of the pattern (asymmetric decoders allowed, SAT; c-1 UNSAT, c SAT)
+  x_proxy = c*h/v      #620's full-rank-totals premise (each total charged the window), now with the exact minrank
+  cst_lab              star cost attainable by label decoders on the B0 labels: min over bases {p_j} of F2^c0 of
+                       sum_j dim_Q span{lines T : u_T . p_j = 1} (matroid greedy, exact); an UPPER bound on cst_min
+  X*                   certified ceiling of #595 (skel.X, rational intervals) maximised over odd s in 3..15
+
+What changed against #620:
+  * E6 was built as the complement of two ORTHOGONAL roots (= D6, 30 lines); it is now the complement of an A2 (36 lines).
+  * the column 'c' was rank_F2(J+G) of one decoder; it is now the exact minrank (both are printed).
+  * an attainable label-decoder star cost is added next to the proxy.
+Usage: python ade_screen_fixed.py REPO_ROOT [--json out.json]
+"""
+import json
+import sys
+import time
+from fractions import Fraction as Fr
+
+import numpy as np
+
+REPO = sys.argv[1]
+sys.path.insert(0, REPO + '/research/format-limits/scripts')
+sys.dont_write_bytecode = True
+from skel import X, profile             # noqa: E402  (#595, certified ceiling)
+from cert import cost_iv                # noqa: E402
+from minrank_sat import solve_asym      # noqa: E402  (#595 SAT encoding, GL(c) gauge on a clique)
+from e8units import sym_factor_f2       # noqa: E402
+
+import units as U                       # noqa: E402
+
+S_RANGE = range(3, 17, 2)
+TIMEOUT = 600                           # seconds per SAT query (A8, D9 hit it: minrank left as a range)
+
+
+def xstar(h, x):
+    best = None
+    for s in S_RANGE:
+        r = X(s, h, 1, Fr(x))
+        if r and (best is None or r['hi'] > best[1]):
+            best = (s, r['hi'], r['lo'])
+    return best
+
+
+def minrank(G, omega_set, c0):
+    lo, hi, log = U.minrank_f2(G, omega_set, c0, solve_asym, timeout=TIMEOUT)
+    return (lo if lo == hi else '%d..%d' % (lo, hi)), log, lo
+
+
+def label_cost(L, G):
+    B0 = (1 + G) % 2
+    Uf = sym_factor_f2(B0).astype(np.int64)
+    c0 = Uf.shape[1]
+    costs = []
+    for p in range(1, 1 << c0):
+        pv = np.array([(p >> j) & 1 for j in range(c0)])
+        R = np.nonzero((Uf @ pv) % 2)[0]
+        costs.append((U.rank_q(L[R].tolist()), p))
+    costs.sort()
+    basis, tot = {}, 0
+    for cst, p in costs:
+        x = p
+        while x:
+            hb = x.bit_length() - 1
+            if hb in basis:
+                x ^= basis[hb]
+            else:
+                basis[hb] = x
+                break
+        if x:
+            tot += cst
+        if len(basis) == c0:
+            break
+    return tot, c0
+
+
+def unit(name, L, G=None):
+    if G is None:
+        G = U.gram_doubled(L)
+    v = len(L)
+    h = U.rank_q(G.tolist())
+    c0 = U.f2_rank((1 + G) % 2)
+    om = U.nonorth_clique(G)
+    c, log, clo = minrank(G, om, c0)
+    cst_lab, _ = label_cost(L, G)
+    xp = Fr(clo * h, v)                 # proxy at the certified lower end of the minrank
+    xl = Fr(cst_lab, v)
+    row = dict(unit=name, v=v, h=h, c0=c0, omega=len(om), minrank=c, sat_log={k: v_ for k, v_ in log.items()},
+               x_proxy=str(xp), cst_lab=cst_lab, x_lab=str(xl))
+    for key, x in (('proxy', xp), ('lab', xl)):
+        b = xstar(h, x)
+        row['Xstar_' + key] = None if b is None else dict(s=b[0], hi=str(b[1]), hi_float=float(b[1]))
+    if row['Xstar_lab'] is None and xl < 1:
+        # positive deficit only at s > 1/(1 - x): bound the ceiling by D_s / Cost_s (#595 A.2) for odd s up to 801
+        best = None
+        for s_ in range(3, 802, 2):
+            prof, m, D = profile(s_, h, 1, xl)
+            if D <= 0:
+                continue
+            b = D / cost_iv(prof, m)[0]
+            if best is None or b > best[1]:
+                best = (s_, b)
+        row['DoverCost_lab'] = dict(s=best[0], hi=str(best[1]), hi_float=float(best[1]))
+        print('   %s: x_lab < 1, deficit > 0 only for s > %s; max over odd s <= 801 of D_s/Cost_s = %.4e at s = %d'
+              % (name, 1 / (1 - xl), float(best[1]), best[0]))
+    return row
+
+
+def main():
+    rows = []
+    cases = [('A%d' % n, U.roots_A(n)) for n in (4, 5, 6, 7, 8)]
+    cases += [('D%d' % n, U.roots_D(n)) for n in (4, 5, 6, 7, 8, 9, 10, 12)]
+    cases += [('D6(archived "E6")', U.roots_D6_archived()), ('E6', U.roots_E6()), ('E7', U.roots_E7()), ('E8', U.roots_E8())]
+    for name, L in cases:
+        t = time.time()
+        r = unit(name, L)
+        rows.append(r)
+        f = lambda k: ('%.4e@s%d' % (r[k]['hi_float'], r[k]['s'])) if r[k] else 'D<=0'
+        print('%-18s v=%4d h=%2d c0=%2d omega=%2d minrank=%-5s x_proxy=%-7s cst_lab=%4d x_lab=%-8s X*proxy=%-14s X*lab=%-14s (%.0fs)'
+              % (name, r['v'], r['h'], r['c0'], r['omega'], r['minrank'], r['x_proxy'], r['cst_lab'], r['x_lab'],
+                 f('Xstar_proxy'), f('Xstar_lab'), time.time() - t), flush=True)
+    if '--json' in sys.argv:
+        json.dump(rows, open(sys.argv[sys.argv.index('--json') + 1], 'w'), indent=1, default=str)
+
+
+if __name__ == '__main__':
+    main()

research/e8-decoder-floors-and-cold14-notes/r1-cold14/REPORT.md

@@ -0,0 +1,18 @@
+# R1 resume (record track) — no strict gain yet
+Worktrees: ~/imb-r1 (branch research/cold14-e2, base pr615, WIP commit with tools in research/r1-wip-tools/),
+~/imb-r1b (branch research/cold14-b3-r1, base pr617, unchanged). Sources: ~/imb-src/pr{568,569,571,575,583}.
+Verify: PATH=$SP/venv/bin:$PATH sh research/<pkg>/verify.sh WORK ~/imb-src  (system python3 is too old). #615 baseline PASS kappa 0.001626010676080313 (615 s).
+Tools ($SP/r1/tools == research/r1-wip-tools): price.py (exact b via PR568 moment.certify), V.json/V617.json (mass-neutral 1e-9 b per rank-r child),
+nulls.py/broad2.py/hypsurvey.py/cands.py (cone capacity), coal.py/coalf.py/movecoal.py (frame-coalescence scans), aglp*.py (aggregate LP), mip617.py + expand617.py (free-dispatch MIP -> OWNERS/PLAN).
+## Findings (on #615 words; #617 keeps E1)
+- e2 infeasible: for all 3 Lean pairs (E=[30],[6],[24] raw) left-nullity of row-cone matrix over same-E helpers == #already-ended (12/24/12) on jp8 and atm3. Cold-mixing directions exist only via cold helpers 542/543/554/555 (E30), 566/567/578/579 (E24) but their column cone is unfixable (col-null space has no vector on them); other violators (315,347, data ports) are blocked by monotonicity.
+- Cold dim-8 (bot,U) extras: best single end net -5.67 (V units), all negative.
+- E1 partners 438/440 cannot be replaced by already-lifted registers (nullity drops 12->11 per dropped partner).
+- Frame coalescence (PR583-type) on jp8 joint-prefix word: 0 single-gate (incl. final frames), 0 with commuting moves W=6 (validation on atm3 reproduces PR583's 6).
+- #615 owner/dispatch: aggregate LP (any remainders, even unlimited windows) == current solution -> no owner/dispatch gain on #615.
+- #617 (kappa 0.001635183729255778): its MIP hit time limit with remainders {1:24,2:2,3:2,7:2}. My LP (any remainders) bound is +119.3e-9 b over #617's map at fractional dispatch jp8 60.9/atm3 11.1 -> an integer perfect packing at jp8x61/atm3x11 would be ~+115e-9 b (~+1.1e-7 kappa). THIS IS THE NEXT LEVER.
+## Resume
+cd $SP/r1/tools && ../../venv/bin/python mip617.py 3 1800 ../m617   # free-dispatch Hall-mask MIP (was running, 900 s)
+mkdir ../m617-out && ../../venv/bin/python expand617.py ../m617.pkl ../m617-out
+../../venv/bin/python price.py "$(cat ../m617-profile.json)"   # compare to b of #617 = 0.0016378619344417451
+Then: copy OWNERS/PLAN into ~/imb-r1b/research/complex-571-cold14-b3/data, update SPEC.json (data hashes, dispatch, expected global_profile/provider_counts/kappa/idle remainders), WITNESS/EXPECTED (spec.py generate), MANIFEST, rerun verify.sh with --reuse-baseline/--reuse-local from $SP/r1/b3-base for iteration, then full fresh run.

research/e8-decoder-floors-and-cold14-notes/r1-cold14/aglp.py

@@ -0,0 +1,80 @@
+"""aggregate free-dispatch LP (first-order, mass-neutral values V) over providers; patterns with any remainder."""
+import json,sys,itertools
+import numpy as np
+from collections import Counter,defaultdict
+from scipy.optimize import linprog
+from scipy.sparse import coo_matrix
+V={int(k):v for k,v in json.load(open('V.json')).items()}
+PLAN=json.load(open('/Users/joelpulikkan/imb-r1/research/complex-571-cold14-e1/data/PLAN.json'))
+reg=PLAN['provider_registry']
+provs=sys.argv[1].split(',') if len(sys.argv)>1 else ['jp8','atm3']
+fixed=json.loads(sys.argv[2]) if len(sys.argv)>2 else None   # {time:provider} optional
+inv={p:Counter(reg[p]['widths']) for p in provs}
+val={p:sum(V[int(r)]*c for r,c in reg[p]['actual_paid_profile'].items()) for p in provs}
+WID=sorted({w for p in provs for w in inv[p]})
+print('values',val,'widths',WID)
+def pats(cap,L):
+    out=[]
+    def rec(pre,i,tot):
+        out.append(tuple(pre))
+        for j in range(i,len(WID)):
+            w=WID[j]
+            if tot+w<=cap and len(pre)<L: rec(pre+[w],j,tot+w)
+    rec([],0,0);return out
+cols=[];cost=[];rows=defaultdict(dict);rhs_eq={};ub_rows=defaultdict(dict);ub={}
+def var(c):
+    cols.append(c);cost.append(-c);return len(cols)-1
+sel={}
+for t in range(72):
+    for p in provs:
+        if fixed and fixed.get(str(t))!=p: continue
+        sel[t,p]=var(val[p])
+    rows[('sel',t)].update({sel[t,p]:1 for p in provs if (t,p) in sel});rhs_eq[('sel',t)]=1
+pv={}
+for key,(cap,times) in PLAN['windows'].items():
+    ps=pats(cap,len(times))
+    for q in ps:
+        i=var(V[cap-sum(q)]);pv[key,q]=i;rows[('ports',key)][i]=1
+    rhs_eq[('ports',key)]=112
+    for t in times:
+        for w in WID:
+            i=var(0);rows[('flow',key,w)][i]=1;rows[('inv',t,w)][i]=1
+            ub_rows[('tcap',key,t)][i]=1;ub[('tcap',key,t)]=112
+    for w in WID:
+        for q in ps:
+            c=q.count(w)
+            if c: rows[('flow',key,w)][pv[key,q]]=-c
+        rhs_eq[('flow',key,w)]=0
+for t in range(72):
+    for w in WID:
+        for p in provs:
+            if (t,p) in sel and inv[p][w]: rows[('inv',t,w)][sel[t,p]]=-inv[p][w]
+        rhs_eq[('inv',t,w)]=0
+def mat(R,keys):
+    r,c,d=[],[],[]
+    for i,k in enumerate(keys):
+        for j,x in R[k].items():r.append(i);c.append(j);d.append(x)
+    return coo_matrix((d,(r,c)),shape=(len(keys),len(cols))).tocsr()
+ek=list(rows);uk=list(ub_rows)
+res=linprog(cost,A_ub=mat(ub_rows,uk),b_ub=[ub[k] for k in uk],A_eq=mat(rows,ek),b_eq=[rhs_eq[k] for k in ek],bounds=(0,None),method='highs')
+print(res.status,res.message,'objective',-res.fun,'vars',len(cols))
+x=res.x
+disp=Counter();
+for (t,p),i in sel.items(): disp[p]+=x[i]
+print('dispatch',dict(disp))
+rem=Counter()
+for (key,q),i in pv.items():
+    if x[i]>1e-7: rem[PLAN['windows'][key][0]-sum(q)]+=x[i]
+print('remainders',{k:round(v,3) for k,v in sorted(rem.items())})
+cur=68*val.get('jp8',0)+4*val.get('atm3',0)
+print('current solution value',cur,'LP gain over current (1e-9 b)',-res.fun-cur)
+json.dump({'sel':{f'{t},{p}':x[i] for (t,p),i in sel.items()},'pv':{f'{k}|{list(q)}':x[i] for (k,q),i in pv.items() if x[i]>1e-9}},open('aglp-out.json','w'))
+du=res.eqlin.marginals
+d=defaultdict(dict)
+for i,k in enumerate(ek):
+    if k[0]=='inv': d[k[2]][k[1]]=-du[i]
+for w in WID:
+    vals=[d[w][t] for t in range(72)]
+    print('dual width',w,'min %.2f max %.2f mean %.2f'%(min(vals),max(vals),sum(vals)/72))
+print('per-time dual(9)-dual(8):',[round(d[9][t]-d[8][t],2) for t in range(72)])
+print('per-time dual(8)-dual(7):',[round(d[8][t]-d[7][t],2) for t in range(72)])

research/e8-decoder-floors-and-cold14-notes/r1-cold14/aglp617.py

@@ -0,0 +1,80 @@
+"""aggregate free-dispatch LP (first-order, mass-neutral values V) over providers; patterns with any remainder."""
+import json,sys,itertools
+import numpy as np
+from collections import Counter,defaultdict
+from scipy.optimize import linprog
+from scipy.sparse import coo_matrix
+V={int(k):v for k,v in json.load(open('V617.json')).items()}
+PLAN=json.load(open('/Users/joelpulikkan/imb-r1b/research/complex-571-cold14-b3/data/PLAN.json'))
+reg=PLAN['provider_registry']
+provs=sys.argv[1].split(',') if len(sys.argv)>1 else ['jp8','atm3']
+fixed=json.loads(sys.argv[2]) if len(sys.argv)>2 else None   # {time:provider} optional
+inv={p:Counter(reg[p]['widths']) for p in provs}
+val={p:sum(V[int(r)]*c for r,c in reg[p]['actual_paid_profile'].items()) for p in provs}
+WID=sorted({w for p in provs for w in inv[p]})
+print('values',val,'widths',WID)
+def pats(cap,L):
+    out=[]
+    def rec(pre,i,tot):
+        out.append(tuple(pre))
+        for j in range(i,len(WID)):
+            w=WID[j]
+            if tot+w<=cap and len(pre)<L: rec(pre+[w],j,tot+w)
+    rec([],0,0);return out
+cols=[];cost=[];rows=defaultdict(dict);rhs_eq={};ub_rows=defaultdict(dict);ub={}
+def var(c):
+    cols.append(c);cost.append(-c);return len(cols)-1
+sel={}
+for t in range(72):
+    for p in provs:
+        if fixed and fixed.get(str(t))!=p: continue
+        sel[t,p]=var(val[p])
+    rows[('sel',t)].update({sel[t,p]:1 for p in provs if (t,p) in sel});rhs_eq[('sel',t)]=1
+pv={}
+for key,(cap,times) in PLAN['windows'].items():
+    ps=pats(cap,len(times))
+    for q in ps:
+        i=var(V[cap-sum(q)]);pv[key,q]=i;rows[('ports',key)][i]=1
+    rhs_eq[('ports',key)]=112
+    for t in times:
+        for w in WID:
+            i=var(0);rows[('flow',key,w)][i]=1;rows[('inv',t,w)][i]=1
+            ub_rows[('tcap',key,t)][i]=1;ub[('tcap',key,t)]=112
+    for w in WID:
+        for q in ps:
+            c=q.count(w)
+            if c: rows[('flow',key,w)][pv[key,q]]=-c
+        rhs_eq[('flow',key,w)]=0
+for t in range(72):
+    for w in WID:
+        for p in provs:
+            if (t,p) in sel and inv[p][w]: rows[('inv',t,w)][sel[t,p]]=-inv[p][w]
+        rhs_eq[('inv',t,w)]=0
+def mat(R,keys):
+    r,c,d=[],[],[]
+    for i,k in enumerate(keys):
+        for j,x in R[k].items():r.append(i);c.append(j);d.append(x)
+    return coo_matrix((d,(r,c)),shape=(len(keys),len(cols))).tocsr()
+ek=list(rows);uk=list(ub_rows)
+res=linprog(cost,A_ub=mat(ub_rows,uk),b_ub=[ub[k] for k in uk],A_eq=mat(rows,ek),b_eq=[rhs_eq[k] for k in ek],bounds=(0,None),method='highs')
+print(res.status,res.message,'objective',-res.fun,'vars',len(cols))
+x=res.x
+disp=Counter();
+for (t,p),i in sel.items(): disp[p]+=x[i]
+print('dispatch',dict(disp))
+rem=Counter()
+for (key,q),i in pv.items():
+    if x[i]>1e-7: rem[PLAN['windows'][key][0]-sum(q)]+=x[i]
+print('remainders',{k:round(v,3) for k,v in sorted(rem.items())})
+cur=60*val.get('jp8',0)+12*val.get('atm3',0)+V[1]*24+V[2]*2+V[3]*2+V[7]*2
+print('current solution value',cur,'LP gain over current (1e-9 b)',-res.fun-cur)
+json.dump({'sel':{f'{t},{p}':x[i] for (t,p),i in sel.items()},'pv':{f'{k}|{list(q)}':x[i] for (k,q),i in pv.items() if x[i]>1e-9}},open('aglp617-out.json','w'))
+du=res.eqlin.marginals
+d=defaultdict(dict)
+for i,k in enumerate(ek):
+    if k[0]=='inv': d[k[2]][k[1]]=-du[i]
+for w in WID:
+    vals=[d[w][t] for t in range(72)]
+    print('dual width',w,'min %.2f max %.2f mean %.2f'%(min(vals),max(vals),sum(vals)/72))
+print('per-time dual(9)-dual(8):',[round(d[9][t]-d[8][t],2) for t in range(72)])
+print('per-time dual(8)-dual(7):',[round(d[8][t]-d[7][t],2) for t in range(72)])