diff --git a/src/components/Scene.svelte b/src/components/Scene.svelte index 9065cea28..c3526ac37 100644 --- a/src/components/Scene.svelte +++ b/src/components/Scene.svelte @@ -339,13 +339,15 @@ // kept rendering finger spheres). shouldSendHands forces it on a rep flip. const lens = [left?.joints?.length ?? 0, right?.joints?.length ?? 0]; if (!shouldSendHands({ moved, hasJoints, prevLens: lastSentJointLens, lens })) return; + // 60 Q2: a hands<->controllers switch is state (reliable); a pose of the same rep is stream + const sameRep = !!lastSentJointLens && lastSentJointLens[0] === lens[0] && lastSentJointLens[1] === lens[1]; lastSentJointLens = lens; lastHandsSendAt = now; for (let i = 0; i < 2; i++) { const pose = poses[i]; if (pose) lastHandPositions[i].fromArray(pose.pos); } - $peers.send({ type: 'vrhands', peerId: $peers.peer.id, left, right, active: true }); + $peers.send({ type: 'vrhands', peerId: $peers.peer.id, left, right, active: true }, { fast: sameRep }); } useTask((delta) => { @@ -414,7 +416,9 @@ nowMs - lastCameraSendAt >= camGapMs) { lastCameraSendAt = nowMs; camContentEuler.setFromQuaternion(camContentQuat); - $peers.send({ type: 'camera', peerId: $peers.peer.id, position: camContentPos.toArray(), rotation: [camContentEuler.x, camContentEuler.y, camContentEuler.z], ...camExtras }); + // 60 Q2: a pose of the stream rides the fast channel; a change of the extras (seat, + // feet, knocked…) is state and goes reliable, or a lost datagram could strand it + $peers.send({ type: 'camera', peerId: $peers.peer.id, position: camContentPos.toArray(), rotation: [camContentEuler.x, camContentEuler.y, camContentEuler.z], ...camExtras }, { fast: camExtrasKey === lastCameraExtras }); lastCameraExtras = camExtrasKey; lastCameraPosition.copy(camContentPos); lastCameraQuaternion.copy(camContentQuat); @@ -1732,13 +1736,14 @@ dropToSurface($TControls.object, $objectsGroup); } $TControls.visible = true; + // 60 Q2: the gizmo drag is a stream (fast); its keyframe lands the last pose $peers.send({ type: 'move', uuid: $TControls.object.uuid, pos: $TControls.object.position.toArray(), rot: $TControls.object.rotation.toArray(), scale: $TControls.object.scale.toArray() - }); + }, { fast: true }); } } diff --git a/src/components/menu/Settings.svelte b/src/components/menu/Settings.svelte index 32df297f6..65d03db7e 100644 --- a/src/components/menu/Settings.svelte +++ b/src/components/menu/Settings.svelte @@ -48,6 +48,7 @@ import ShortcutsSettings from './settings/ShortcutsSettings.svelte'; import SceneSettings from './settings/SceneSettings.svelte'; // checkpoints, loading, water render inside import ExplorerSettings from './settings/ExplorerSettings.svelte'; + import StorageSettings from './settings/StorageSettings.svelte'; // 60 Q6: the local asset cache import NodeTypesSection from './NodeTypesSection.svelte'; // 36 B7 // 36-export: the export defaults (its own file; also the Publish / Export modal's Settings tab) import ExportSettingsSection from './ExportSettingsSection.svelte'; @@ -84,6 +85,8 @@ $: if ((settingsQuery || '').trim() || ($settingsOpen && $settingsSection === 'accessibility')) accessibilityExpanded = true; let sceneExpanded = false; let explorerExpanded = false; + let storageExpanded = false; // 60 Q6: Settings ▸ Storage (StorageSettings.svelte) + $: if ((settingsQuery || '').trim() || ($settingsOpen && $settingsSection === 'storage')) storageExpanded = true; let connectionExpanded = false; let aboutExpanded = false; let vrExpanded = false; // D7: edit-cap toasts deep-link here ('vr') @@ -562,6 +565,10 @@ {#snippet header()}Explorer{/snippet} + + {#snippet header()}Storage{/snippet} + + {#snippet header()}VR{/snippet} diff --git a/src/components/menu/settings/StorageSettings.svelte b/src/components/menu/settings/StorageSettings.svelte new file mode 100644 index 000000000..ebf30e7aa --- /dev/null +++ b/src/components/menu/settings/StorageSettings.svelte @@ -0,0 +1,167 @@ + + + + +
+
+ + {#snippet desc()} + + {#if $assetCacheBackend === 'none'} + This browser gives the page no storage to keep files in, so everything is downloaded + each time. + {:else} + {fmtBytes(usage.bytes)} in {usage.blobs} + {usage.blobs === 1 ? 'file' : 'files'}{parts ? ' — ' + parts : ''}. Packs, skies, + converted textures and downloads are kept so they load again without the network. The + app itself is never kept here, so updates always arrive. + {/if} + {#if cleared} {cleared}{/if} + + {/snippet} + + + + assetCacheCapMB.set(Number(v))} + /> + + + {#snippet desc()} + + {#if $persistState === 'granted'} + The browser keeps this storage even when it is short of space. + {:else if $persistState === 'unsupported'} + This browser decides by itself when to clear site storage. + {:else} + The browser may clear the cache when it is short of space. Asking lets it keep it; some + browsers decide by how often you visit. + {/if} + + {/snippet} + {#if $persistState === 'not-granted'} + + {/if} + + + + +
+
+ + diff --git a/src/lib/animatedImports.js b/src/lib/animatedImports.js index d3cb6e020..87e77a6b6 100644 --- a/src/lib/animatedImports.js +++ b/src/lib/animatedImports.js @@ -481,10 +481,12 @@ async function fetchAnimRef(ref) { const url = packRefUrl(ref); let job = refFiles.get(url); if (!job) { - job = fetch(url).then((res) => { - if (!res.ok) throw new Error('HTTP ' + res.status + ' ' + url); - return res.arrayBuffer(); - }); + // 60 Q6: a pack file like any other — through the pack cache (the local asset cache) + job = import('./packCache').then((m) => + m.fetchPackBuffer(url).catch((/** @type {any} */ e) => { + throw new Error(String(e?.message ?? e) + ' ' + url); + }) + ); refFiles.set(url, job); job.catch(() => refFiles.delete(url)); } diff --git a/src/lib/assetCache.js b/src/lib/assetCache.js new file mode 100644 index 000000000..783c99266 --- /dev/null +++ b/src/lib/assetCache.js @@ -0,0 +1,454 @@ +import { writable, get } from 'svelte/store'; +import { safeStorage } from './safeStorage'; +import { + emptyIndex, + normalizeIndex, + usageOf, + planEviction, + orphanedBy, + urlKey, + CAP_CHOICES_MB, + DEFAULT_CAP_MB +} from './assetCacheCore.js'; + +// 60 Q6 — THE LOCAL ASSET CACHE: packs, skies, converted textures and downloaded imports, kept +// on this device in the Origin Private File System, keyed by CONTENT HASH (assetCacheCore.js +// has the model: keys -> hash, one blob per hash, LRU under a cap). +// +// WHY OPFS. 39 P5 kept pack files in CacheStorage, which can be listed and sized but stores a +// whole Response per url — the same bytes at two urls twice — and gives a page no way to read a +// file without materialising the Response. OPFS is a real per-origin filesystem: a blob per hash, +// one small index file, a byte count that is the file's own. Where OPFS cannot write (no +// `createWritable`: old Safari, a non-secure origin) the SAME model runs on CacheStorage, and +// where neither exists every call degrades to a plain network fetch, so no caller ever asks. +// +// WHAT IT NEVER HOLDS: the app shell. index.html, the JS/CSS chunks and the service worker stay +// under the no-cache SW rule — an update must always reach the next load. Only CONTENT reaches +// this module: pack files from the packs CDN, sky images, CC0 downloads, conversion outputs. +// +// PERSISTENCE: browsers may evict an origin's storage under pressure unless the page asked +// `navigator.storage.persist()`. It is asked ONCE, from a toast that says why, the first time +// something lands here (`assetCache:persistAsked`), never on boot and never again. +// +// A LEAF for the app (svelte/store + safeStorage + the pure core); the toast is a dynamic import. + +const DIR = 'tp-asset-cache'; +const CACHE_NAME = 'tp-asset-cache-v1'; +const CACHE_BASE = '/__tp-asset-cache/'; +const INDEX_FLUSH_MS = 400; +const CAP_KEY = 'assetCache:capMB'; +const ASKED_KEY = 'assetCache:persistAsked'; + +/** @typedef {{name: string, read(hash: string): Promise, write(hash: string, buf: ArrayBuffer): Promise, remove(hash: string): Promise, list(): Promise, readIndex(): Promise, writeIndex(text: string): Promise, wipe(): Promise}} Backend */ + +/** @returns {Promise} */ +async function opfsBackend() { + const storage = /** @type {any} */ (typeof navigator !== 'undefined' ? navigator.storage : null); + if (!storage || typeof storage.getDirectory !== 'function') return null; + const root = await storage.getDirectory(); + const dir = await root.getDirectoryHandle(DIR, { create: true }); + const blobs = await dir.getDirectoryHandle('blobs', { create: true }); + const probe = await dir.getFileHandle('.probe', { create: true }); + if (typeof probe.createWritable !== 'function') return null; + /** @param {any} handle @param {ArrayBuffer | string} data */ + const writeFile = async (handle, data) => { + const w = await handle.createWritable(); + await w.write(data); + await w.close(); + }; + return { + name: 'opfs', + async read(hash) { + try { + const fh = await blobs.getFileHandle(hash); + return await (await fh.getFile()).arrayBuffer(); + } catch { + return null; + } + }, + async write(hash, buf) { + await writeFile(await blobs.getFileHandle(hash, { create: true }), buf); + }, + async remove(hash) { + try { + await blobs.removeEntry(hash); + } catch {} + }, + async list() { + /** @type {string[]} */ + const out = []; + for await (const name of blobs.keys()) out.push(name); + return out; + }, + async readIndex() { + try { + const fh = await dir.getFileHandle('index.json'); + return JSON.parse(await (await fh.getFile()).text()); + } catch { + return null; + } + }, + async writeIndex(text) { + await writeFile(await dir.getFileHandle('index.json', { create: true }), text); + }, + async wipe() { + for await (const name of blobs.keys()) { + try { + await blobs.removeEntry(name); + } catch {} + } + } + }; +} + +/** the same model on CacheStorage (one Response per blob, one for the index) @returns {Promise} */ +async function cacheBackend() { + if (typeof caches === 'undefined') return null; + const cache = await caches.open(CACHE_NAME); + const blobUrl = (/** @type {string} */ hash) => CACHE_BASE + 'blob/' + hash; + return { + name: 'cachestorage', + async read(hash) { + const res = await cache.match(blobUrl(hash)); + return res ? res.arrayBuffer() : null; + }, + async write(hash, buf) { + await cache.put(blobUrl(hash), new Response(buf.slice(0))); + }, + async remove(hash) { + await cache.delete(blobUrl(hash)); + }, + async list() { + return (await cache.keys()) + .map((r) => new URL(r.url).pathname) + .filter((p) => p.startsWith(CACHE_BASE + 'blob/')) + .map((p) => p.slice((CACHE_BASE + 'blob/').length)); + }, + async readIndex() { + const res = await cache.match(CACHE_BASE + 'index.json'); + return res ? res.json() : null; + }, + async writeIndex(text) { + await cache.put( + CACHE_BASE + 'index.json', + new Response(text, { headers: { 'content-type': 'application/json' } }) + ); + }, + async wipe() { + for (const req of await cache.keys()) await cache.delete(req); + } + }; +} + +/** @type {Promise | null} */ +let backendJob = null; +/** @type {Backend | null} */ +let backend = null; +let index = emptyIndex(); +/** the index, republished on every change (Settings, Storage, packCache read it) */ +export const assetCacheIndex = writable(index); +/** which store is in use: 'opfs' | 'cachestorage' | 'none' | '' (not known yet) */ +export const assetCacheBackend = writable(''); + +/** @returns {number} */ +function readCapMB() { + const n = Number(safeStorage.getItem(CAP_KEY)); + return CAP_CHOICES_MB.includes(n) ? n : DEFAULT_CAP_MB; +} +/** the LRU cap in MB (Settings ▸ Storage); LOCAL, persisted */ +export const assetCacheCapMB = writable(readCapMB()); +let capRead = false; +assetCacheCapMB.subscribe((mb) => { + if (!capRead) return void (capRead = true); // the value we just read is not a choice + if (CAP_CHOICES_MB.includes(mb)) safeStorage.setItem(CAP_KEY, String(mb)); + if (backend) void enforceCap(); +}); + +const stats = { hits: 0, misses: 0, writes: 0, evicted: 0, fetches: 0 }; + +/** Open the store and read the index, once. Resolves whether a store is usable. @returns {Promise} */ +export function assetCacheReady() { + if (!backendJob) { + backendJob = (async () => { + /** @type {Backend | null} */ + let b = null; + try { + b = await opfsBackend(); + } catch { + b = null; + } + if (!b) { + try { + b = await cacheBackend(); + } catch { + b = null; + } + } + if (b) { + try { + index = normalizeIndex(await b.readIndex()); + } catch { + index = emptyIndex(); + } + // a tab that died between a blob write and its index flush leaves a blob no key + // names, and one that died mid-eviction a key whose blob is gone: reconcile both + try { + const held = new Set(await b.list()); + let dropped = false; + for (const [k, e] of Object.entries(index.entries)) { + if (!held.has(e.hash)) { + delete index.entries[k]; + dropped = true; + } + } + const named = new Set(Object.values(index.entries).map((e) => e.hash)); + for (const h of held) if (!named.has(h)) await b.remove(h); + if (dropped) scheduleFlush(); + } catch {} + } + backend = b; + assetCacheBackend.set(b ? b.name : 'none'); + publish(); + return b; + })(); + } + return backendJob.then((b) => !!b); +} + +function publish() { + assetCacheIndex.set(index); +} + +/** @type {ReturnType | null} */ +let flushTimer = null; +function scheduleFlush() { + if (flushTimer) return; + flushTimer = setTimeout(() => { + flushTimer = null; + void flushIndex(); + }, INDEX_FLUSH_MS); +} +/** Write the index now (also on pagehide). */ +export async function flushIndex() { + if (!backend) return; + try { + await backend.writeIndex(JSON.stringify(index)); + } catch {} +} +if (typeof addEventListener === 'function') addEventListener('pagehide', () => void flushIndex()); + +/** SHA-256 hex of the bytes, or null where SubtleCrypto is missing @param {ArrayBuffer} buf */ +async function sha256(buf) { + try { + const digest = await crypto.subtle.digest('SHA-256', buf); + return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, '0')).join(''); + } catch { + return null; + } +} + +/** Is `key` held (by the index; the bytes may still turn out gone)? @param {string} key */ +export function hasCached(key) { + return !!index.entries[key]; +} + +/** The index row of `key`, or null @param {string} key */ +export function cachedEntry(key) { + return index.entries[key] ?? null; +} + +/** + * The cached bytes of `key`, or null. A hit stamps the key as just used (LRU). + * @param {string | null} key @returns {Promise} + */ +export async function readCached(key) { + if (!key || !(await assetCacheReady()) || !backend) return null; + const e = index.entries[key]; + if (!e) { + stats.misses++; + return null; + } + const buf = await backend.read(e.hash); + if (!buf) { + // the blob went (another tab cleared, the browser evicted): forget the row + delete index.entries[key]; + publish(); + scheduleFlush(); + stats.misses++; + return null; + } + e.lastUsed = Date.now(); + scheduleFlush(); + stats.hits++; + return buf; +} + +/** + * Keep `buffer` under `key`. Best effort: a full quota or a refused store leaves the app as it + * was (the bytes are already in memory). Resolves the content hash, or null when nothing was kept. + * @param {string | null} key @param {ArrayBuffer} buffer + * @param {{kind: string, type?: string, meta?: any}} info + * @returns {Promise} + */ +export async function writeCached(key, buffer, info) { + if (!key || !buffer || !(await assetCacheReady()) || !backend) return null; + const hash = await sha256(buffer); + if (!hash) return null; + const shared = Object.values(index.entries).some((e) => e.hash === hash); + try { + if (!shared) await backend.write(hash, buffer); + } catch { + return null; + } + const now = Date.now(); + index.entries[key] = { + hash, + bytes: buffer.byteLength, + kind: info.kind, + ...(info.type ? { type: info.type } : {}), + lastUsed: now, + at: now, + ...(info.meta !== undefined ? { meta: info.meta } : {}) + }; + stats.writes++; + await enforceCap([key]); + publish(); + scheduleFlush(); + maybeOfferPersist(); + return hash; +} + +/** Drop the least recently used keys until the cache fits its cap. @param {string[]} [protect] */ +async function enforceCap(protect = []) { + if (!backend) return; + const plan = planEviction(index, get(assetCacheCapMB) * 1024 * 1024, protect); + if (!plan.keys.length) return; + for (const k of plan.keys) delete index.entries[k]; + for (const h of plan.hashes) await backend.remove(h); + stats.evicted += plan.keys.length; + publish(); + scheduleFlush(); +} + +/** + * The bytes of a network file through the cache: the held copy when there is one (no network + * at all), else a fetch whose bytes are kept. Throws like fetch does on a failed request. + * @param {string} url @param {{kind: string, version?: string, init?: RequestInit}} opts + * @returns {Promise} + */ +export async function fetchCached(url, opts) { + const key = urlKey(url, opts.version); + const held = await readCached(key); + if (held) return held; + stats.fetches++; + const res = await fetch(url, opts.init); + if (!res.ok) throw Object.assign(new Error('HTTP ' + res.status), { status: res.status }); + const buffer = await res.arrayBuffer(); + void writeCached(key, buffer, { + kind: opts.kind, + type: res.headers.get('content-type') || undefined + }); + return buffer; +} + +/** + * Forget `keys`; blobs no other key names are deleted. Resolves the bytes freed. + * @param {string[]} keys @returns {Promise} + */ +export async function deleteCached(keys) { + if (!(await assetCacheReady()) || !backend) return 0; + const orphans = orphanedBy(index, keys); + let freed = 0; + for (const h of orphans) { + const e = Object.values(index.entries).find((x) => x.hash === h); + freed += e?.bytes ?? 0; + await backend.remove(h); + } + for (const k of keys) delete index.entries[k]; + publish(); + scheduleFlush(); + return freed; +} + +/** Settings ▸ Storage ▸ Clear: everything, every kind. Resolves the bytes freed. */ +export async function clearAssetCache() { + if (!(await assetCacheReady()) || !backend) return 0; + const freed = usageOf(index).bytes; + try { + await backend.wipe(); + } catch {} + index = emptyIndex(); + publish(); + await flushIndex(); + return freed; +} + +// ---- persistence: asked once, with the reason ------------------------------------------------ + +/** 'granted' | 'not-granted' | 'unsupported' | '' (not read yet) */ +export const persistState = writable(''); + +export async function refreshPersistState() { + const storage = /** @type {any} */ (typeof navigator !== 'undefined' ? navigator.storage : null); + if (!storage || typeof storage.persisted !== 'function') { + persistState.set('unsupported'); + return; + } + try { + persistState.set((await storage.persisted()) ? 'granted' : 'not-granted'); + } catch { + persistState.set('unsupported'); + } +} + +/** Was the question put already (on this device)? */ +export function persistAsked() { + return safeStorage.getItem(ASKED_KEY) === '1'; +} + +/** Ask the browser to keep this origin's storage. The ONE call site of `persist()`. */ +export async function requestPersist() { + safeStorage.setItem(ASKED_KEY, '1'); + const storage = /** @type {any} */ (typeof navigator !== 'undefined' ? navigator.storage : null); + if (!storage || typeof storage.persist !== 'function') { + persistState.set('unsupported'); + return false; + } + let ok = false; + try { + ok = await storage.persist(); + } catch {} + persistState.set(ok ? 'granted' : 'not-granted'); + return ok; +} + +let offered = false; +/** The first time something lands in the cache, say why we want to keep it — once, ever. */ +function maybeOfferPersist() { + if (offered || persistAsked()) return; + offered = true; + void (async () => { + await refreshPersistState(); + if (get(persistState) !== 'not-granted') return; // granted already, or no such API + safeStorage.setItem(ASKED_KEY, '1'); + const { showToastById } = await import('../stores/appStore.js'); + showToastById( + 'asset-cache-persist', + 'Downloaded packs and skies are kept on this device so scenes load faster and work offline. Keep them even when the browser is short of space?', + [ + { label: 'Keep them', action: () => void requestPersist() }, + { label: 'Not now', action: () => {} } + ] + ); + })(); +} + +/** for the suites (`__stores.assetCache`) */ +export function assetCacheStats() { + return { + ...stats, + backend: backend?.name ?? 'none', + ...usageOf(index), + capMB: get(assetCacheCapMB) + }; +} diff --git a/src/lib/assetCacheCore.js b/src/lib/assetCacheCore.js new file mode 100644 index 000000000..23a2ccc4b --- /dev/null +++ b/src/lib/assetCacheCore.js @@ -0,0 +1,190 @@ +// 60 Q6 — THE LOCAL ASSET CACHE, its pure half: keys, the index shape and the LRU plan. +// +// A ZERO-IMPORT LEAF (the wireBatch / alignCore shape), so every rule that decides WHAT is kept +// and WHAT goes is unit-testable with no OPFS and no browser. `assetCache.js` is the runtime +// that reads and writes bytes; this file only does arithmetic over the index. +// +// THE MODEL. Bytes are stored ONCE per content hash (a "blob"); the index maps a KEY to the hash +// it resolved to. Two keys may share a blob — the same file reached at two urls, or a converted +// texture identical to one already held — and a blob is freed only when no key refers to it. +// * url:[#v=] — a network file (pack files, skies, CC0 downloads). The +// hash is learned when the bytes arrive; the key is what a later load asks for. A version tag +// is part of the key where the url alone does not pin the bytes (a same-origin sky ships with +// the build, so it is keyed by the build too). +// * derived::: — the OUTPUT of a conversion (an optimized import's +// KTX2 textures), keyed by the INPUT's content hash and the exact settings. The same file +// imported again with the same settings never converts twice. +// +// LRU: every read stamps `lastUsed`; when the blobs' total passes the cap, keys go oldest first +// (never the protected ones — the key just written) until the blobs still referenced fit. + +/** @typedef {{hash: string, bytes: number, kind: string, type?: string, lastUsed: number, at: number, meta?: any}} CacheEntry */ +/** @typedef {{v: 1, entries: Record}} CacheIndex */ + +export const INDEX_VERSION = 1; +export const CACHE_KINDS = ['pack', 'sky', 'texture', 'import']; +/** the cap choices Settings offers, in MB (0 is not offered: Clear is the off switch) */ +export const CAP_CHOICES_MB = [256, 512, 1024, 2048]; +export const DEFAULT_CAP_MB = 1024; + +/** + * JSON with SORTED keys at every depth, so two equal settings objects give one key whatever + * order their fields were written in. + * @param {any} value @returns {string} + */ +export function stableStringify(value) { + if (value === undefined) return 'null'; + if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null'; + if (Array.isArray(value)) return '[' + value.map((v) => stableStringify(v)).join(',') + ']'; + const keys = Object.keys(value) + .filter((k) => value[k] !== undefined) + .sort(); + return '{' + keys.map((k) => JSON.stringify(k) + ':' + stableStringify(value[k])).join(',') + '}'; +} + +/** + * The key of a network file. Only absolute http(s) urls are cacheable — a blob:/data: url is + * already in memory, and a relative one must be resolved by the caller first. + * @param {string} url @param {string} [version] @returns {string | null} + */ +export function urlKey(url, version) { + if (typeof url !== 'string' || !/^https?:\/\//i.test(url)) return null; + const bare = url.replace(/#.*$/, ''); + return 'url:' + bare + (version ? '#v=' + version : ''); +} + +/** the url a `url:` key was made from (without its version tag), else null @param {string} key */ +export function urlOfKey(key) { + if (typeof key !== 'string' || !key.startsWith('url:')) return null; + return key.slice(4).replace(/#v=.*$/, ''); +} + +/** + * The key of a conversion's output. + * @param {string} kind @param {string} sourceHash @param {any} params @returns {string | null} + */ +export function derivedKey(kind, sourceHash, params) { + if (!kind || typeof sourceHash !== 'string' || !/^[0-9a-f]{16,}$/i.test(sourceHash)) return null; + return 'derived:' + kind + ':' + sourceHash.toLowerCase() + ':' + stableStringify(params ?? null); +} + +/** @returns {CacheIndex} */ +export function emptyIndex() { + return { v: INDEX_VERSION, entries: {} }; +} + +/** + * Bring a stored index back to shape: an unknown version or a malformed row is dropped (the + * cache is a cache — losing a row costs one download), never trusted. + * @param {any} raw @returns {CacheIndex} + */ +export function normalizeIndex(raw) { + const out = emptyIndex(); + if ( + !raw || + typeof raw !== 'object' || + raw.v !== INDEX_VERSION || + !raw.entries || + typeof raw.entries !== 'object' + ) + return out; + for (const [key, e] of Object.entries(raw.entries)) { + if (!e || typeof e !== 'object') continue; + if (typeof e.hash !== 'string' || !/^[0-9a-f]{16,}$/.test(e.hash)) continue; + if (!Number.isFinite(e.bytes) || e.bytes < 0) continue; + out.entries[key] = { + hash: e.hash, + bytes: e.bytes, + kind: CACHE_KINDS.includes(e.kind) ? e.kind : 'import', + ...(typeof e.type === 'string' ? { type: e.type } : {}), + lastUsed: Number.isFinite(e.lastUsed) ? e.lastUsed : 0, + at: Number.isFinite(e.at) ? e.at : 0, + ...(e.meta !== undefined ? { meta: e.meta } : {}) + }; + } + return out; +} + +/** + * The blobs the index refers to: hash -> {bytes, refs}. A blob's size is counted ONCE however + * many keys share it. + * @param {CacheIndex} index @returns {Map} + */ +export function blobsOf(index) { + /** @type {Map} */ + const blobs = new Map(); + for (const e of Object.values(index.entries)) { + const b = blobs.get(e.hash); + if (b) b.refs++; + else blobs.set(e.hash, { bytes: e.bytes, refs: 1 }); + } + return blobs; +} + +/** + * What the cache holds: total bytes (blobs counted once), keys, blobs, and bytes per kind (a + * shared blob counts toward the kind of the first key that names it, so the kinds sum to the + * total). + * @param {CacheIndex} index + */ +export function usageOf(index) { + /** @type {Record} */ + const byKind = Object.fromEntries(CACHE_KINDS.map((k) => [k, 0])); + const seen = new Set(); + let bytes = 0; + for (const e of Object.values(index.entries)) { + if (seen.has(e.hash)) continue; + seen.add(e.hash); + bytes += e.bytes; + byKind[e.kind] = (byKind[e.kind] ?? 0) + e.bytes; + } + return { bytes, keys: Object.keys(index.entries).length, blobs: seen.size, byKind }; +} + +/** + * THE LRU PLAN: which keys to drop so the referenced blobs fit `capBytes`, oldest `lastUsed` + * first (ties by key, so the plan is deterministic), never a `protect`ed key; and which blobs + * that frees (a blob goes only when its LAST key goes). Pure — the caller applies it. + * @param {CacheIndex} index @param {number} capBytes @param {Iterable} [protect] + * @returns {{keys: string[], hashes: string[], bytesAfter: number}} + */ +export function planEviction(index, capBytes, protect = []) { + const keep = new Set(protect); + const blobs = blobsOf(index); + let total = 0; + for (const b of blobs.values()) total += b.bytes; + /** @type {string[]} */ + const keys = []; + /** @type {string[]} */ + const hashes = []; + if (total <= capBytes) return { keys, hashes, bytesAfter: total }; + const order = Object.entries(index.entries) + .filter(([k]) => !keep.has(k)) + .sort((a, b) => a[1].lastUsed - b[1].lastUsed || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)); + for (const [key, e] of order) { + if (total <= capBytes) break; + keys.push(key); + const b = /** @type {{bytes: number, refs: number}} */ (blobs.get(e.hash)); + b.refs--; + if (b.refs === 0) { + hashes.push(e.hash); + total -= b.bytes; + } + } + return { keys, hashes, bytesAfter: total }; +} + +/** + * Which blobs no key refers to any more once `keys` are removed (the delete/clear-one path). + * @param {CacheIndex} index @param {Iterable} keys @returns {string[]} + */ +export function orphanedBy(index, keys) { + const gone = new Set(keys); + const still = new Set(); + const touched = new Set(); + for (const [k, e] of Object.entries(index.entries)) { + if (gone.has(k)) touched.add(e.hash); + else still.add(e.hash); + } + return [...touched].filter((h) => !still.has(h)); +} diff --git a/src/lib/cc0/cc0Fetch.js b/src/lib/cc0/cc0Fetch.js index 51cd8080c..702f351eb 100644 --- a/src/lib/cc0/cc0Fetch.js +++ b/src/lib/cc0/cc0Fetch.js @@ -5,11 +5,13 @@ // one IndexedDB store under `cc0:` with the time it arrived. A fresh entry answers with no // network at all; an expired one is refetched, and when that refetch FAILS (offline, the site // down, the relay down) the old answer is served with `stale: true`, so the browser keeps working -// on what it saw last and says so. Downloads are NOT cached here — they become Library items, -// which are content-hashed and deduped by the import entry already. +// on what it saw last and says so. Downloads become Library items (content-hashed and deduped by +// the import entry); 60 Q6 also keeps their BYTES in the local asset cache (`fetchBytes`). import { idbGet, idbPut } from '../idb'; import { safeStorage } from '../safeStorage'; import { PH_API, ACG_PAGE, DEFAULT_CC0_PROXY, classifyError } from './cc0Core.js'; +import { readCached, writeCached } from '../assetCache'; +import { urlKey } from '../assetCacheCore.js'; const HOUR = 3600_000; /** how long an answer is fresh, per endpoint family */ @@ -110,6 +112,21 @@ export async function acgResolve(file) { * @returns {Promise} */ export async function fetchBytes(url, onProgress, signal) { + // 60 Q6: a download is kept in the local asset cache (kind `import`), so importing the same + // file again — after deleting it from the library, or into another project — costs no network + const key = urlKey(url); + const held = await readCached(key); + if (held) { + onProgress?.(held.byteLength, held.byteLength); + return new Uint8Array(held); + } + const bytes = await downloadBytes(url, onProgress, signal); + void writeCached(key, /** @type {ArrayBuffer} */ (bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength)), { kind: 'import' }); + return bytes; +} + +/** @param {string} url @param {(done: number, total: number) => void} [onProgress] @param {AbortSignal} [signal] @returns {Promise} */ +async function downloadBytes(url, onProgress, signal) { let res; try { res = await fetch(url, { credentials: 'omit', signal }); diff --git a/src/lib/cloudHooks.js b/src/lib/cloudHooks.js index 5b09cd835..fa3cb43b8 100644 --- a/src/lib/cloudHooks.js +++ b/src/lib/cloudHooks.js @@ -31,6 +31,9 @@ const ALWAYS_ALLOWED = new Set([ 'getnodedefs', 'getjoints', 'getchat', // 37 R15: the chat-history REQUEST (its reply stays gateable, the getnodes rule) + // 60 Q2: "I opened the fast movement channel on this connection" — transport, no content. A + // gated hello would leave a viewer's avatar on the slow channel while the host thinks it is fast + 'fastchan', // A1/A2: WHERE A PEER IS STANDING. On the floor because it is the ROOM GATE'S OWN // EVIDENCE - a plugin that gated `atscene` would silently switch off scene adoption // AND every room gate built on it (`canApplyByRoom` reads no evidence, so it allows diff --git a/src/lib/debugHooks.js b/src/lib/debugHooks.js index 46aa9ba1b..0ccf384f9 100644 --- a/src/lib/debugHooks.js +++ b/src/lib/debugHooks.js @@ -43,6 +43,8 @@ export const DEBUG_HOOKS = [ ['placementPrefs', () => import('./placementPrefs.js')], ['placementDims', () => import('./placementDims.js')], ['packCache', () => import('./packCache.js')], + // 60 Q6: the local asset cache (OPFS) — index, stats, clear, persist + ['assetCache', () => import('./assetCache.js')], ['placeGhost', () => import('./placeGhost.js')], ['tourVR', () => import('./tours/vrPanel.js')], // 36-share (B13): the recorder + its stores diff --git a/src/lib/fastChannel.js b/src/lib/fastChannel.js new file mode 100644 index 000000000..3b84c9020 --- /dev/null +++ b/src/lib/fastChannel.js @@ -0,0 +1,292 @@ +// 60 Q2 — THE FAST MOVEMENT CHANNEL. +// +// Every message used to ride ONE PeerJS DataConnection per peer, which is ordered and reliable. +// That is what an edit needs and the opposite of what a pose stream needs: on a lossy link one +// lost packet holds back EVERYTHING behind it until SCTP retransmits it (head-of-line blocking), +// so an avatar that is sent 20 times a second freezes for a retransmit timeout and then jumps +// through a burst of poses that were all stale by the time they arrived. +// +// So each pair of peers opens a SECOND channel on the RTCPeerConnection they already share, +// `{ordered: false, maxRetransmits: 0}`: a lost pose is simply gone, and the next one (50 ms +// later) replaces it. Only latest-wins pose messages go there — avatars (`camera`), VR hands +// (`vrhands`) and the STREAMED `move`s of a drag, a held object or a physics body. Everything +// else — every edit, every committed move, every request — stays on the reliable channel. +// +// FOUR RULES: +// * A NEGOTIATED channel (`negotiated: true, id: FAST_CHANNEL_ID`) on the PeerJS connection's +// own RTCPeerConnection: no extra signaling, no second ICE run, no renegotiation (an SCTP +// association that exists takes more streams without one). Both sides must create it, so it +// is created only after the OTHER side said it can (`fc: 1` in the `modules` handshake, then +// a `fastchan` hello down the same conn so both ends pick the same RTCPeerConnection). +// * GRACEFUL FALLBACK: a peer that never said `fc: 1` (an older build), a channel that is not +// open yet, a message too big for one unreliable datagram — all go down the reliable conn +// exactly as before. The wire to an older peer is byte-identical except one additive field. +// * SEQUENCE NUMBERS: every pose message carries `ps`, one monotonic counter per sender. A +// receiver keeps the newest `ps` per (sender, key) and drops anything older — so an +// unordered datagram never moves an avatar backwards, and a late streamed `move` never +// overwrites the committed (reliable, newer) one that ended the drag. +// * A TRAILING KEYFRAME: the LAST pose of a stream may be the one that is lost, and nothing +// would ever correct it (a camera that stopped moving stops sending). So when a key goes +// quiet after a fast send, its last payload is re-sent once on the reliable channel with +// the SAME `ps` — dropped as a duplicate where the datagram arrived, applied where it didn't. +// +// Kill switch: `safeStorage.fastChannel = 'off'` (the e2e's "before" and "older build"). +import { safeStorage } from './safeStorage'; + +/** the SCTP stream id both ends give the negotiated channel (PeerJS's own channel is 0 or 1) */ +export const FAST_CHANNEL_ID = 7; +export const FAST_CHANNEL_LABEL = 'tp-fast'; +/** above this a message goes reliable: one unreliable message should fit a few datagrams, and a + * lost fragment loses the whole message */ +export const FAST_MAX_BYTES = 12000; +/** past this many queued bytes a pose is SKIPPED, not queued: behind a backlog it would only arrive + * stale, and the next pose (or the trailing keyframe) replaces it anyway */ +export const FAST_MAX_BUFFERED = 256 * 1024; +/** how long a key must be quiet before its trailing keyframe goes out */ +export const KEYFRAME_MS = 250; + +/** The pose types and how each is keyed: a camera/hands stream per sender, a move per object. */ +export const POSE_TYPES = new Set(['camera', 'vrhands', 'move']); + +/** @param {any} payload @returns {string | null} */ +export function poseKey(payload) { + const type = payload?.type; + if (type === 'camera' || type === 'vrhands') return type; + if (type === 'move' && typeof payload.uuid === 'string') return 'move:' + payload.uuid; + return null; +} + +/** Is the channel allowed on this device? @returns {boolean} */ +export function fastChannelEnabled() { + try { + return safeStorage.getItem('fastChannel') !== 'off'; + } catch { + return true; + } +} + +// Seeded from the clock (x100) so a RELOADED sender (same peer id, fresh counter) still starts +// above everything its previous page sent — up to 100k pose messages a second before it could +// outrun the clock. Receivers also forget a peer on disconnect. +let seq = Date.now() * 100; +/** @returns {number} */ +export function nextPoseSeq() { + return ++seq; +} + +/** + * Give a pose payload its `ps` (once — a keyframe re-send keeps the original). Non-pose + * payloads come back untouched. Returns the SAME object, stamped. + * @param {any} payload @returns {any} + */ +export function stampPose(payload) { + if ( + payload && + typeof payload === 'object' && + POSE_TYPES.has(payload.type) && + !Number.isFinite(payload.ps) + ) + payload.ps = nextPoseSeq(); + return payload; +} + +/** @type {((peerId: string, payload: any, via: 'fast' | 'reliable') => void) | null} */ +let poseTap = null; +/** + * A MEASUREMENT seam (the loss e2e): `fn` sees every pose that passed the stale filter, i.e. + * every pose the app applies, with the channel it came on. Null (the default) costs one check. + * @param {((peerId: string, payload: any, via: 'fast' | 'reliable') => void) | null} fn + */ +export function setPoseTap(fn) { + poseTap = typeof fn === 'function' ? fn : null; +} +/** @param {string} peerId @param {any} payload @param {'fast' | 'reliable'} via */ +export function tapPose(peerId, payload, via) { + if (poseTap && poseKey(payload) !== null) poseTap(peerId, payload, via); +} + +/** + * The receive-side stale filter. `accept` answers false for a pose older than (or equal to) + * the newest already applied from that sender for that key. A payload with no `ps` (an older + * build) is always accepted — and records nothing. + */ +export function createStaleFilter() { + /** `${peer}|${key}` -> newest ps @type {Map} */ + const newest = new Map(); + const stats = { accepted: 0, stale: 0 }; + return { + stats, + /** @param {string} peerId @param {any} payload @returns {boolean} */ + accept(peerId, payload) { + const ps = payload?.ps; + if (!Number.isFinite(ps)) return true; + const key = poseKey(payload); + if (!key) return true; + const id = peerId + '|' + key; + const last = newest.get(id); + if (last !== undefined && ps <= last) { + stats.stale++; + return false; + } + newest.set(id, ps); + stats.accepted++; + return true; + }, + /** @param {string} peerId */ + dropPeer(peerId) { + const prefix = peerId + '|'; + for (const id of [...newest.keys()]) if (id.startsWith(prefix)) newest.delete(id); + }, + clear() { + newest.clear(); + }, + size: () => newest.size + }; +} + +/** + * The trailing keyframes. `noteFast` (re)arms a key's timer with its newest payload and the + * peers that got it unreliably; `noteReliable` disarms it (a reliable send of the same key + * supersedes it). When a timer fires, `resend(payload, peerIds)` sends it on the reliable path. + * @param {(payload: any, peerIds: string[]) => void} resend + * @param {{delay?: number, setTimer?: (fn: () => void, ms: number) => any, clearTimer?: (t: any) => void}} [opts] + */ +export function createKeyframer(resend, opts = {}) { + const delay = opts.delay ?? KEYFRAME_MS; + const setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms)); + const clearTimer = opts.clearTimer ?? ((t) => clearTimeout(t)); + /** key -> {timer, payload, peers} @type {Map}>} */ + const pending = new Map(); + const stats = { armed: 0, sent: 0 }; + return { + stats, + /** @param {string} key @param {any} payload @param {Iterable} peerIds */ + noteFast(key, payload, peerIds) { + const held = pending.get(key); + if (held) clearTimer(held.timer); + const peers = new Set(peerIds); + const entry = { payload, peers, timer: null }; + entry.timer = setTimer(() => { + if (pending.get(key) !== entry) return; + pending.delete(key); + stats.sent++; + resend(entry.payload, [...entry.peers]); + }, delay); + pending.set(key, entry); + stats.armed++; + }, + /** @param {string} key */ + noteReliable(key) { + const held = pending.get(key); + if (!held) return; + clearTimer(held.timer); + pending.delete(key); + }, + /** a peer left: it needs no keyframes @param {string} peerId */ + dropPeer(peerId) { + for (const entry of pending.values()) entry.peers.delete(peerId); + }, + clear() { + for (const entry of pending.values()) clearTimer(entry.timer); + pending.clear(); + }, + size: () => pending.size + }; +} + +/** + * The channels themselves, one per peer. `open(peerId, conn, onMessage)` creates the negotiated + * channel on `conn.peerConnection` (idempotent per RTCPeerConnection; a newer conn replaces an + * older one's channel). `trySend` answers false whenever the reliable path must carry it. + */ +export function createFastLinks() { + /** peerId -> {dc, pc} @type {Map} */ + const links = new Map(); + const stats = { opened: 0, sent: 0, received: 0, tooBig: 0, congested: 0, sendErrors: 0, parseErrors: 0 }; + /** @param {string} peerId */ + function close(peerId) { + const link = links.get(peerId); + if (!link) return; + links.delete(peerId); + try { + link.dc.close(); + } catch {} + } + return { + stats, + /** + * @param {string} peerId @param {any} conn a PeerJS DataConnection + * @param {(data: any, via: 'fast') => void} onMessage @returns {boolean} true when a channel exists now + */ + open(peerId, conn, onMessage) { + const pc = conn?.peerConnection; + if (!pc || typeof pc.createDataChannel !== 'function') return false; + const held = links.get(peerId); + if (held && held.pc === pc && held.dc.readyState !== 'closed') return true; + if (held) close(peerId); + let dc; + try { + dc = pc.createDataChannel(FAST_CHANNEL_LABEL, { + ordered: false, + maxRetransmits: 0, + negotiated: true, + id: FAST_CHANNEL_ID + }); + } catch { + return false; + } + const link = { dc, pc }; + links.set(peerId, link); + dc.onopen = () => { + stats.opened++; + }; + dc.onclose = () => { + if (links.get(peerId) === link) links.delete(peerId); + }; + dc.onmessage = (/** @type {MessageEvent} */ e) => { + if (typeof e.data !== 'string') return; + let data; + try { + data = JSON.parse(e.data); + } catch { + stats.parseErrors++; + return; + } + stats.received++; + onMessage(data, 'fast'); + }; + return true; + }, + /** @param {string} peerId @returns {boolean} */ + isOpen(peerId) { + return links.get(peerId)?.dc.readyState === 'open'; + }, + /** @param {string} peerId @param {string} text an already-serialized payload @returns {boolean} */ + trySend(peerId, text) { + const link = links.get(peerId); + if (!link || link.dc.readyState !== 'open') return false; + if (text.length > FAST_MAX_BYTES) { + stats.tooBig++; + return false; + } + // a backlog: skip this pose (it counts as sent unreliably — a datagram lost early) + if (link.dc.bufferedAmount > FAST_MAX_BUFFERED) { + stats.congested++; + return true; + } + try { + link.dc.send(text); + stats.sent++; + return true; + } catch { + stats.sendErrors++; + return false; + } + }, + close, + closeAll() { + for (const peerId of [...links.keys()]) close(peerId); + }, + peers: () => [...links.keys()] + }; +} diff --git a/src/lib/hdri/hdriRuntime.js b/src/lib/hdri/hdriRuntime.js index 846075bf2..9c9e497e2 100644 --- a/src/lib/hdri/hdriRuntime.js +++ b/src/lib/hdri/hdriRuntime.js @@ -17,6 +17,8 @@ import { hdriOf, resolveHdriTier, isHeadsetUA, TIER_MAX_WIDTH, downsampleRGBA, i import { bundledHdri } from './catalog.js'; import { skyEnv, envToneMapping, hdriStatus } from './skyEnv.js'; import { pageUrl } from '../export/exportBoot.js'; +import { fetchCached } from '../assetCache'; +import { APP_VERSION } from '../version'; import { qualityOverrides } from '../qualityGovernor'; import { isVRMode } from '../../stores/sceneStore'; import { hdriQuality } from './hdriPrefs.js'; @@ -72,9 +74,15 @@ async function fetchSource(src) { if (held) return held; const row = bundledHdri(src); if (row) { - const res = await fetch(pageUrl(row.file)); - if (!res.ok) throw new Error('HTTP ' + res.status + ' for ' + row.file); - const got = { buffer: await res.arrayBuffer(), name: row.file }; + // 60 Q6: through the local asset cache (kind `sky`). A bundled sky ships WITH the build, so + // its key carries the build's version: an updated file under the same name is fetched anew + let buffer; + try { + buffer = await fetchCached(pageUrl(row.file), { kind: 'sky', version: APP_VERSION }); + } catch (e) { + throw new Error(String(/** @type {any} */ (e)?.message ?? e) + ' for ' + row.file); + } + const got = { buffer, name: row.file }; remember(sources, src, got); return got; } diff --git a/src/lib/importOptimize.js b/src/lib/importOptimize.js index 3384b0b68..47cd46cf5 100644 --- a/src/lib/importOptimize.js +++ b/src/lib/importOptimize.js @@ -14,6 +14,10 @@ import { retainOriginal } from './decimate'; import { generatedGroup } from './lodGroupCore'; import { parseGltfSceneSafe } from './ktx2Twins'; import { optimizeGltf, OptimizeUnsupported } from './gltfOptimize.js'; +// 60 Q6: the converted output is kept in the local asset cache, keyed by the source hash + settings +import { readCached, writeCached, cachedEntry } from './assetCache'; +import { derivedKey } from './assetCacheCore.js'; +import { APP_VERSION } from './version'; import { importDecision, importOptions, @@ -157,34 +161,53 @@ export async function maybeOptimizeImport(parsed, source) { /** @type {any} */ let report = null; let optimizedBytes = null; + // 60 Q6: the same file under the same settings converts ONCE per build — the textures' + // resize + KTX2 encode is the slow part of an import, and its output is a pure function of both + const engineOptions = importOptions(settings); + const convertedKey = runEngine + ? derivedKey('texture', hash, { options: engineOptions, v: APP_VERSION }) + : null; + const converted = convertedKey ? await readCached(convertedKey) : null; if (runEngine) { const progressId = 'import-optimize-' + hash.slice(0, 12); - showInfoToast( - progressId, - 'Optimizing "' + source.label + '"… you can keep working while it runs.', - [], - undefined, - true - ); + if (!converted) + showInfoToast( + progressId, + 'Optimizing "' + source.label + '"… you can keep working while it runs.', + [], + undefined, + true + ); try { - const out = await optimizeGltf(bytes, importOptions(settings), { - onProgress: (p) => { - if (p.step === 'textures' && p.total) - showInfoToast( - progressId, - 'Optimizing "' + - source.label + - '": textures ' + - Math.min(p.total, (p.done ?? 0) + 1) + - ' / ' + - p.total + - '…', - [], - undefined, - true - ); - } - }); + const out = converted + ? { + bytes: new Uint8Array(converted), + report: cachedEntry(/** @type {string} */ (convertedKey))?.meta?.report ?? null + } + : await optimizeGltf(bytes, engineOptions, { + onProgress: (p) => { + if (p.step === 'textures' && p.total) + showInfoToast( + progressId, + 'Optimizing "' + + source.label + + '": textures ' + + Math.min(p.total, (p.done ?? 0) + 1) + + ' / ' + + p.total + + '…', + [], + undefined, + true + ); + } + }); + if (!converted && convertedKey) + void writeCached(convertedKey, out.bytes.slice().buffer, { + kind: 'texture', + type: 'model/gltf-binary', + meta: { report: out.report } + }); report = out.report; optimizedBytes = out.bytes; const gltfResult = await parseGltfSceneSafe( @@ -254,7 +277,9 @@ export async function maybeOptimizeImport(parsed, source) { optimizedFrom: original.hash, optimizeReport: summaryForItem, // 47 C2: the optimized copy names the same CC0 source as its original (Explorer Properties) - ...(/** @type {any} */ (original).source ? { source: /** @type {any} */ (original).source } : {}) + .../** @type {any} */ ((original).source + ? { source: /** @type {any} */ (original).source } + : {}) }); patchRecord(original.id, { optimizedHash: optimizedItem.hash }); } diff --git a/src/lib/packCache.js b/src/lib/packCache.js index 72f41132c..f2bebb62c 100644 --- a/src/lib/packCache.js +++ b/src/lib/packCache.js @@ -1,71 +1,77 @@ -import { writable, get } from 'svelte/store'; +import { derived, get } from 'svelte/store'; +import { + assetCacheIndex, + assetCacheReady, + readCached, + writeCached, + deleteCached, + fetchCached +} from './assetCache'; +import { urlKey, urlOfKey } from './assetCacheCore.js'; -// 39 P5 — PACK DOWNLOADS: the files of pack items this device has fetched, kept in ONE -// CacheStorage cache so the Explorer can say which items are downloaded (and how big they -// are), "Delete cache" can free one item's bytes while the item stays in its pack, and -// Storage can clear them all. +// 39 P5 — PACK DOWNLOADS: the files of pack items this device has fetched, so the Explorer can +// say which items are downloaded (and how big they are), "Delete cache" can free one item's bytes +// while the item stays in its pack, and Storage can clear them all. // -// Before this every pack file went through the browser's HTTP cache alone — invisible to -// the page, evictable at any time, and impossible to list or delete. CacheStorage is the -// one store a page can both enumerate and size. The HTTP cache still sits underneath (we -// cannot clear it), so "Delete cache" frees OUR copy: the next fetch goes to the network -// (or whatever the browser still holds) and is cached here again. +// 60 Q6: the bytes live in the LOCAL ASSET CACHE now (assetCache.js — OPFS, keyed by content +// hash, LRU under the Settings ▸ Storage cap) as kind `pack`. This module keeps its API — every +// caller (placement, scene pack refs, the Explorer badges, Storage) is unchanged — and is the +// url-keyed VIEW of that cache. The 39 P5 CacheStorage copies are moved over once +// (`migrateLegacy`) and that cache is deleted. // -// A LEAF: svelte/store only. LOCAL by construction — what a device has downloaded is a fact -// about this machine; nothing here travels, saves or undoes. Every call degrades to a plain -// network fetch where CacheStorage is missing (an exported game on file://, a non-secure -// origin, a private window that refuses it), so no caller ever has to ask. +// LOCAL by construction — what a device has downloaded is a fact about this machine; nothing +// here travels, saves or undoes. Every call degrades to a plain network fetch where no store +// works (an exported game on file://, a non-secure origin, a private window that refuses it). +/** 39 P5's CacheStorage name — only read once, to migrate what it holds */ export const PACK_CACHE_NAME = 'tp-pack-files-v1'; -/** the size header written beside each cached response (a Response's body cannot be sized - * without reading it, and the index must not read every blob on boot) */ -const BYTES_HEADER = 'x-tp-bytes'; -/** url -> cached bytes, for every file in the cache. The Explorer's badges and the Storage - * row read it; `loadPackCacheIndex` fills it once, every put/delete keeps it current. - * @type {import('svelte/store').Writable>} */ -export const packCacheIndex = writable(new Map()); -let indexLoaded = false; -/** @type {Promise | null} */ -let indexJob = null; +/** url -> cached bytes, for every pack file held. The Explorer's badges and the Storage row read + * it; it follows the asset cache's index. @type {import('svelte/store').Readable>} */ +export const packCacheIndex = derived(assetCacheIndex, (idx) => { + /** @type {Map} */ + const map = new Map(); + for (const [key, e] of Object.entries(idx.entries)) { + if (e.kind !== 'pack') continue; + const url = urlOfKey(key); + if (url) map.set(url, e.bytes); + } + return map; +}); /** Only real network files are cached — a blob:/data: url is already in memory. @param {string} url */ export function cacheableUrl(url) { return typeof url === 'string' && /^https?:\/\//.test(url); } -/** @returns {Promise} */ -async function openCache() { - try { - if (typeof caches === 'undefined') return null; - return await caches.open(PACK_CACHE_NAME); - } catch { - return null; - } +/** @type {Promise | null} */ +let migrateJob = null; +/** 60 Q6: move 39 P5's CacheStorage pack copies into the asset cache, then delete that cache. */ +function migrateLegacy() { + if (migrateJob) return migrateJob; + migrateJob = (async () => { + try { + if (typeof caches === 'undefined' || !(await caches.has(PACK_CACHE_NAME))) return; + const old = await caches.open(PACK_CACHE_NAME); + for (const req of await old.keys()) { + const res = await old.match(req); + if (!res) continue; + const buffer = await res.arrayBuffer(); + await writeCached(urlKey(req.url), buffer, { + kind: 'pack', + type: res.headers.get('content-type') || undefined + }); + } + await caches.delete(PACK_CACHE_NAME); + } catch {} + })(); + return migrateJob; } -/** Read the cache's keys (and sizes) into `packCacheIndex`, once. */ -export function loadPackCacheIndex() { - if (indexLoaded) return Promise.resolve(); - if (indexJob) return indexJob; - indexJob = (async () => { - const cache = await openCache(); - /** @type {Map} */ - const map = new Map(); - if (cache) { - try { - for (const req of await cache.keys()) { - const res = await cache.match(req); - const bytes = Number(res?.headers.get(BYTES_HEADER)) || 0; - map.set(req.url, bytes); - } - } catch {} - } - // a put that raced the scan is already in the store — keep it - packCacheIndex.update((live) => new Map([...map, ...live])); - indexLoaded = true; - })(); - return indexJob; +/** Open the cache and read its index (and migrate the old one), once. */ +export async function loadPackCacheIndex() { + await assetCacheReady(); + await migrateLegacy(); } /** @param {string} url @returns {boolean} */ @@ -86,95 +92,50 @@ export function cachedState(url, index = get(packCacheIndex)) { } /** - * The cached bytes of `url`, or null when it is not downloaded (or the cache is unusable). + * The cached bytes of `url`, or null when it is not downloaded (or no store works). * @param {string} url @returns {Promise} */ export async function cachedPackBuffer(url) { if (!cacheableUrl(url)) return null; - const cache = await openCache(); - if (!cache) return null; - try { - const res = await cache.match(url); - if (!res) { - // a stale index row (cleared in another tab) — forget it - if (get(packCacheIndex).has(url)) packCacheIndex.update((m) => (m.delete(url), new Map(m))); - return null; - } - return await res.arrayBuffer(); - } catch { - return null; - } + await loadPackCacheIndex(); + return readCached(urlKey(url)); } /** - * Keep `buffer` as the downloaded copy of `url`. Best effort: a full quota or a refused - * cache leaves the app exactly as it was (the bytes are already in memory). + * Keep `buffer` as the downloaded copy of `url`. Best effort. * @param {string} url @param {ArrayBuffer} buffer @param {string} [type] */ export async function putPackBuffer(url, buffer, type = 'model/gltf-binary') { if (!cacheableUrl(url) || !buffer) return false; - const cache = await openCache(); - if (!cache) return false; - try { - await cache.put( - url, - new Response(buffer.slice(0), { - headers: { 'content-type': type, [BYTES_HEADER]: String(buffer.byteLength), 'x-tp-cached-at': String(Date.now()) } - }) - ); - packCacheIndex.update((m) => new Map(m).set(url, buffer.byteLength)); - return true; - } catch { - return false; - } + return !!(await writeCached(urlKey(url), buffer, { kind: 'pack', type })); } /** - * The bytes of a pack file: the downloaded copy when there is one, else the network (and - * the result is kept). The one fetch every placement path shares. + * The bytes of a pack file: the downloaded copy when there is one, else the network (and the + * result is kept). The one fetch every placement path shares. * @param {string} url @param {RequestInit} [init] @returns {Promise} */ export async function fetchPackBuffer(url, init) { - const held = await cachedPackBuffer(url); - if (held) return held; - const res = await fetch(url, init); - if (!res.ok) throw Object.assign(new Error('HTTP ' + res.status), { status: res.status }); - const buffer = await res.arrayBuffer(); - void putPackBuffer(url, buffer, res.headers.get('content-type') || undefined); - return buffer; + if (!cacheableUrl(url)) { + const res = await fetch(url, init); + if (!res.ok) throw Object.assign(new Error('HTTP ' + res.status), { status: res.status }); + return res.arrayBuffer(); + } + await loadPackCacheIndex(); + return fetchCached(url, { kind: 'pack', init }); } /** - * Forget the downloaded copies of `urls`. Resolves the bytes freed (by the index's count). + * Forget the downloaded copies of `urls`. Resolves the bytes freed. * @param {string[]} urls @returns {Promise} */ export async function deletePackCache(urls) { - const cache = await openCache(); - let freed = 0; - const index = get(packCacheIndex); - for (const url of urls) { - const bytes = index.get(url) ?? 0; - let gone = false; - try { - gone = cache ? await cache.delete(url) : false; - } catch {} - if (gone || index.has(url)) freed += bytes; - } - packCacheIndex.update((m) => { - const next = new Map(m); - for (const url of urls) next.delete(url); - return next; - }); - return freed; + const keys = urls.map((u) => urlKey(u)).filter((k) => k !== null); + return deleteCached(/** @type {string[]} */ (keys)); } /** "Clear all pack downloads" (Storage). @returns {Promise} the bytes freed */ export async function clearPackCache() { await loadPackCacheIndex(); - const freed = cachedBytesOf([...get(packCacheIndex).keys()]); - try { - if (typeof caches !== 'undefined') await caches.delete(PACK_CACHE_NAME); - } catch {} - packCacheIndex.set(new Map()); - return freed; + return deletePackCache([...get(packCacheIndex).keys()]); } diff --git a/src/lib/peerHandler.svelte.js b/src/lib/peerHandler.svelte.js index 4e194783f..ba0cbd62f 100644 --- a/src/lib/peerHandler.svelte.js +++ b/src/lib/peerHandler.svelte.js @@ -30,6 +30,9 @@ import { canApply, getAuthProvider, dispatchCloudMessage, rolesInfo } from '$lib // dispatcher can reject a malformed message before any applier sees it. import { validateWireMessage } from '$lib/wireValidate'; import { holdForWireBatch } from '$lib/wireBatch'; +// 60 Q2: the fast movement channel (an unordered, unreliable second channel for pose streams) +import { createFastLinks, createStaleFilter, createKeyframer, stampPose, poseKey, fastChannelEnabled, setPoseTap, tapPose } from '$lib/fastChannel'; +import { registerDebugHook } from '$lib/debugHooks'; import { routePerfLive, perfLivePeerGone } from '$lib/perf/liveWire.js'; // 34 PF: the live perf stream (an import-free leaf) import { noteWireError } from '$lib/wireErrors'; import { noteWire } from '$lib/sceneBudget'; @@ -295,6 +298,20 @@ export class PeerConnection { this.dialCloud = new Map(); /** 25-F: peers a refusal dial is out to — their `peer-unavailable` is not news @type {Set} */ this.refusalDials = new Set(); + // 60 Q2: the fast movement channel (fastChannel.js). `fastLinks` = one unreliable channel per + // peer that advertised `fc: 1`; `staleFilter` drops a pose older than the newest applied from + // its sender; `keyframer` re-sends a stream's last pose reliably once the stream goes quiet. + this.fastLinks = createFastLinks(); + this.staleFilter = createStaleFilter(); + this.keyframer = createKeyframer((payload, peerIds) => this.sendReliableTo(payload, peerIds)); + registerDebugHook('fastChannel', { + links: this.fastLinks, + stale: this.staleFilter, + keyframes: this.keyframer, + enabled: fastChannelEnabled, + isOpen: (/** @type {string} */ peerId) => this.fastLinks.isOpen(peerId), + setPoseTap + }); // CN-3: an invite link can pin the signaling world (#A1B2C~srv=…). Parse it // HERE, before resolvePeerOptions runs — the peer.on('open') hash flow below @@ -834,6 +851,7 @@ export class PeerConnection { noteWireError(conn.peer, 'invalid:' + item.type); continue; } + if (!this.staleFilter.accept(conn.peer, item)) continue; // 60 Q2, as for a bare message try { dispatch(item); } catch (error) { @@ -1284,8 +1302,14 @@ export class PeerConnection { // 37 R1: `wb` is ADDITIVE — an older peer omits it and keeps getting bare messages if (data.wb === 1) this.wireBatchPeers.add(conn.peer); else this.wireBatchPeers.delete(conn.peer); + // 60 Q2: `fc` is ADDITIVE too — they can take a fast channel, so open ours on THIS + // conn and say so down it (`fastchan`), so both ends pick the same RTCPeerConnection + if (data.fc === 1 && this.openFastChannel(conn)) conn.send({ type: 'fastchan', v: 1 }); checkModuleVersions(data.versions); checkPeerAppVersion(data.appVersion); + } else if(data.type == 'fastchan') { + // 60 Q2: the other end opened its fast channel on this conn's RTCPeerConnection + this.openFastChannel(conn); } else if(data.type == 'getmodulestate') { sendModuleStates(data.sender); sendCameraPreviewState(); // 16-P5: ride the same late-joiner request @@ -1375,7 +1399,9 @@ export class PeerConnection { } }; - conn.on('data', (data) => { + // 60 Q2: ONE receive path for both channels — the reliable conn and the fast channel + // opened on its RTCPeerConnection (`openFastChannel` reads it off the conn) + const receive = (/** @type {any} */ data, /** @type {'fast' | 'reliable'} */ via = 'reliable') => { // 27-A (audit H1): SHAPE FIRST, before any gate reads `data.type`. A null, a // string or a number used to fall through the whole chain to // `data.startsWith(...)` and throw out of the handler, where peerjs swallowed @@ -1396,13 +1422,20 @@ export class PeerConnection { noteWireError(conn.peer, 'invalid:' + data.type); return; } + // 60 Q2: a pose older than the newest one applied from this sender is stale — an + // unordered datagram overtaken by the next one, or a streamed move that lost the race + // to the committed (reliable, newer) move that ended the drag + if (!this.staleFilter.accept(conn.peer, data)) return; + tapPose(conn.peer, data, via); try { dispatch(data); } catch (error) { // One bad message must not take this connection's handler down with it. noteWireError(conn.peer, data.type, error); } - }); + }; + conn.__tpReceive = receive; + conn.on('data', (/** @type {any} */ data) => receive(data)); } } @@ -1525,7 +1558,7 @@ export class PeerConnection { conn.send({type: 'userdata', userdata: users}) // V3: app version rides the modules handshake — old peers ignore the extras, // old senders omit them (checkPeerAppVersion is silent on absence) - conn.send({type: 'modules', versions: moduleVersions(), appVersion: APP_VERSION, sha: COMMIT_SHA, wb: 1}) + conn.send({type: 'modules', versions: moduleVersions(), appVersion: APP_VERSION, sha: COMMIT_SHA, wb: 1, ...(fastChannelEnabled() ? { fc: 1 } : {})}) // 37 R15: the chat so far. Chat is MESH-WIDE (not a room's content), so this goes out // whatever the connect decision / privacy hold below decides about the scene. conn.send({type: 'getchat', sender: this.peer.id}) @@ -1861,6 +1894,7 @@ export class PeerConnection { try { conn.close(); } catch (e) { /* already dead */ } } this.openedPeers.delete(peerId); + this.dropFastPeer(peerId); // 60 Q2 handleDisconnected(peerId); clearPeerPreview(peerId); // 16-P5 perfLivePeerGone(peerId); // 34 PF @@ -1892,6 +1926,10 @@ export class PeerConnection { this.reconnecting.clear(); // pending retry timers see a cleared map and no-op this.gracefulLeft.clear(); this.wantsStateFrom.clear(); + // 60 Q2: every fast channel, sequence and owed keyframe goes with the session + this.fastLinks.closeAll(); + this.staleFilter.clear(); + this.keyframer.clear(); for (const peerId of Object.keys(this.connections)) { const conn = this.connections[peerId]; delete this.connections[peerId]; @@ -1927,8 +1965,15 @@ export class PeerConnection { // Broadcast to every OPEN outgoing connection. Guards conn.open (peerjs // silently drops pre-open sends) and isolates each send so one dead/half-open // conn can't throw mid-loop and starve the rest of the mesh (172). - /** @param {any} payload */ - broadcast(payload) { + /** + * @param {any} payload + * @param {{fast?: boolean}} [opts] 60 Q2: `fast` = a latest-wins pose of a STREAM (a drag, a held + * object, a physics body, the avatar) that may ride the unreliable fast channel to peers that + * have one. Only pose types honour it (`poseKey`); everything else is reliable whatever it says. + */ + broadcast(payload, opts) { + // 60 Q2: every pose carries its sequence number — batched, fast or reliable alike + stampPose(payload); // 37 R1: inside a multi-object edit, room content is HELD and leaves as one envelope // when the edit closes (wireBatch.js); anything else flushes the held edit first if (holdForWireBatch(payload, ROOM_SCOPED.has(payload?.type), (out) => this.broadcast(out))) return; @@ -1978,6 +2023,13 @@ export class PeerConnection { // would ask round 35's question, which is why it is passed unconditionally. const host = gated ? get(sessionHost) : null; const secret = roomScoped && amPrivate(); + // 60 Q2: which peers took it unreliably (they are owed the trailing keyframe) + const key = poseKey(payload); + const fast = !!opts?.fast && key !== null && fastChannelEnabled(); + /** @type {string | null} */ + let text = null; + /** @type {string[]} */ + const fastPeers = []; Object.keys(this.connections).forEach(peerId => { const conn = this.connections[peerId]; if (!conn || !conn.open) return; @@ -1985,6 +2037,19 @@ export class PeerConnection { if (gated && elsewhereThan(where, mine, peerId, host)) return; if (roomScoped && gateHolds(peerId)) return; try { + if (fast && this.fastLinks.isOpen(peerId)) { + if (text === null) { + try { + text = JSON.stringify(payload); + } catch { + text = ''; + } + } + if (text && this.fastLinks.trySend(peerId, text)) { + fastPeers.push(peerId); + return; + } + } // 37 R1: a peer that never said `wb: 1` gets the envelope's items one by one if (payload?.type === 'batch' && !this.wireBatchPeers.has(peerId)) for (const item of payload.items) conn.send(item); @@ -1993,6 +2058,53 @@ export class PeerConnection { log('error', 'net', 'send failed', { peer: peerId, error: String(err) }); } }); + if (key !== null) { + if (fastPeers.length) this.keyframer.noteFast(key, payload, fastPeers); + else this.keyframer.noteReliable(key); + } + } + + /** + * 60 Q2: the trailing keyframe — a stream's LAST pose, re-sent on the reliable conn to the peers + * that got it unreliably, with its original `ps` (a duplicate where the datagram arrived). + * Re-checks the gates `broadcast` applies: the peer may have walked into another scene since. + * @param {any} payload @param {string[]} peerIds + */ + sendReliableTo(payload, peerIds) { + const mine = myScene()?.scene ?? ''; + const where = get(peerScenes); + const host = get(sessionHost); + for (const peerId of peerIds) { + const conn = this.connections[peerId]; + if (!conn || !conn.open) continue; + if (STREAM_TYPES.has(payload?.type) || ROOM_SCOPED.has(payload?.type)) { + if (elsewhereThan(where, mine, peerId, host)) continue; + } + if (ROOM_SCOPED.has(payload?.type) && (amPrivate() || gateHolds(peerId))) continue; + try { + conn.send(payload); + } catch (err) { + log('error', 'net', 'send failed', { peer: peerId, error: String(err) }); + } + } + } + + /** + * 60 Q2: open the fast channel to `conn.peer` on `conn`'s RTCPeerConnection (idempotent). + * Answers true when it created or already had one there; false when this device has the + * channel switched off or the conn exposes no RTCPeerConnection (the reliable path carries on). + * @param {any} conn @returns {boolean} + */ + openFastChannel(conn) { + if (!fastChannelEnabled() || !conn?.peer || typeof conn.__tpReceive !== 'function') return false; + return this.fastLinks.open(conn.peer, conn, conn.__tpReceive); + } + + /** 60 Q2: forget a peer's fast channel, its sequence numbers and its owed keyframes @param {string} peerId */ + dropFastPeer(peerId) { + this.fastLinks.close(peerId); + this.staleFilter.dropPeer(peerId); + this.keyframer.dropPeer(peerId); } /** @param {string} message @param {string} [type] */ @@ -2009,14 +2121,14 @@ export class PeerConnection { } } - /** @param {any} data */ - send(data) { + /** @param {any} data @param {{fast?: boolean}} [opts] 60 Q2: see `broadcast` */ + send(data, opts) { // viewer perms: a viewer's object CREATIONS never leave this machine — mark // them local-only + warn, and skip the broadcast (peers drop them anyway). if (gateCreationBroadcast(data)) return; if(data.type == 'create') this.sendMessage('created a ' + data.command.split(' ')[1], 'info'); - this.broadcast(data); + this.broadcast(data, opts); } } diff --git a/src/lib/physics.js b/src/lib/physics.js index b03b944bf..4ee66d466 100644 --- a/src/lib/physics.js +++ b/src/lib/physics.js @@ -1935,13 +1935,17 @@ function stepInner(now) { entry.lastSent ??= { pos: new THREE.Vector3(), quat: new THREE.Quaternion() }; entry.lastSent.pos.copy(object.position); entry.lastSent.quat.copy(object.quaternion); - peer.send({ - type: 'move', - uuid: object.uuid, - pos: object.position.toArray(), - rot: [object.rotation.x, object.rotation.y, object.rotation.z], - scale: object.scale.toArray() - }); + // 60 Q2: a body's pose stream rides the fast channel + peer.send( + { + type: 'move', + uuid: object.uuid, + pos: object.position.toArray(), + rot: [object.rotation.x, object.rotation.y, object.rotation.z], + scale: object.scale.toArray() + }, + { fast: true } + ); } } }); diff --git a/src/lib/playInteract.js b/src/lib/playInteract.js index 4cf3925de..ba465f057 100644 --- a/src/lib/playInteract.js +++ b/src/lib/playInteract.js @@ -610,14 +610,18 @@ export function tickPlayInteract(delta, camera) { // deviation detector skips and the write-back excludes, so there is no // fight; when we are not, these moves refresh the EXTERNAL hold, which is // pre-existing behaviour simply fed at a usable rate + // 60 Q2: a held object's stream rides the fast channel if (peer) - peer.send({ - type: 'move', - uuid: grab.object.uuid, - pos: grab.object.position.toArray(), - rot: [grab.object.rotation.x, grab.object.rotation.y, grab.object.rotation.z], - scale: grab.object.scale.toArray() - }); + peer.send( + { + type: 'move', + uuid: grab.object.uuid, + pos: grab.object.position.toArray(), + rot: [grab.object.rotation.x, grab.object.rotation.y, grab.object.rotation.z], + scale: grab.object.scale.toArray() + }, + { fast: true } + ); } pokeScene(); return; diff --git a/src/lib/possess.js b/src/lib/possess.js index f74ca92e5..2cdbb0ad0 100644 --- a/src/lib/possess.js +++ b/src/lib/possess.js @@ -246,14 +246,18 @@ function tick(now) { state.lastSent = now; /** @type {any} */ const peer = get(peers); + // 60 Q2: the ride's pose stream rides the fast channel if (peer) - peer.send({ - type: 'move', - uuid: object.uuid, - pos: object.position.toArray(), - rot: [object.rotation.x, object.rotation.y, object.rotation.z], - scale: object.scale.toArray() - }); + peer.send( + { + type: 'move', + uuid: object.uuid, + pos: object.position.toArray(), + rot: [object.rotation.x, object.rotation.y, object.rotation.z], + scale: object.scale.toArray() + }, + { fast: true } + ); } } diff --git a/src/lib/settings/categories.js b/src/lib/settings/categories.js index 30e4b0356..9b323cd7d 100644 --- a/src/lib/settings/categories.js +++ b/src/lib/settings/categories.js @@ -27,6 +27,7 @@ const META = { shortcuts: { group: 'general', order: 5, description: 'Click a shortcut’s keys, then press the new combination.' }, scene: { group: 'workspace', order: 6, description: 'How the 3D view draws, saves and behaves on this device.' }, explorer: { group: 'workspace', order: 7, description: 'Files, sharing and the recycle bin.' }, + storage: { group: 'workspace', order: 7.5, description: 'Downloads kept on this device for faster, offline loads: size, limit and Clear.' }, // 60 Q6 nodetypes: { group: 'workspace', order: 8, description: 'Hide the node types you never use from the palette, add menus and search on this device.' }, import: { group: 'workspace', order: 8.5, description: 'What happens to big models and textures as they come in.' }, export: { group: 'workspace', order: 9, description: 'What the next exported game starts with.' }, diff --git a/src/lib/storageUsage.js b/src/lib/storageUsage.js index f34eccd5b..161b764ba 100644 --- a/src/lib/storageUsage.js +++ b/src/lib/storageUsage.js @@ -63,6 +63,7 @@ import { clearSavedSession } from './autosave'; import { checkpoints, deleteCheckpoint, KEY as KEY_CHECKPOINT, INDEX_KEY as KEY_CHECKPOINT_INDEX } from './checkpoints'; import { fmtBytes } from './transferLedger'; import { loadPackCacheIndex, packCacheIndex, deletePackCache, clearPackCache } from './packCache'; +import { assetCacheIndex, deleteCached } from './assetCache'; // 60 Q6: the rest of the local asset cache import { clearMeasuredDims } from './placementDims'; import { PACKS_BASE, packs } from './packs'; import { listImageVersions, forgetImageVersions, isImageVersionKey } from './image/imageVersions'; // 40-image @@ -186,6 +187,12 @@ export const CATEGORIES = [ label: 'Pack downloads', note: 'Files of pack items this device downloaded. Clearing them keeps every item in its pack: it downloads again the next time it is placed or a scene needs it.' }, + { + // 60 Q6: the rest of the local asset cache (OPFS) — Settings ▸ Storage has its limit and Clear + key: 'assetcache', + label: 'Cached downloads', + note: 'Skies, converted textures and downloaded files kept on this device so they load again without the network. Clearing them only costs a download (or a conversion) the next time.' + }, { key: 'modules', label: 'Installed modules', @@ -736,6 +743,31 @@ export async function scanStorage() { }); } catch {} + // ---- 60 Q6: the asset cache's other kinds, one row per kind --------------------- + try { + const labels = /** @type {Record} */ ({ sky: 'Skies', texture: 'Converted textures', import: 'Downloaded files' }); + /** @type {Map, bytes: number}>} */ + const byKind = new Map(); + for (const [k, e] of Object.entries(get(assetCacheIndex).entries)) { + if (e.kind === 'pack') continue; // listed per pack above + const row = byKind.get(e.kind) ?? { keys: [], hashes: new Set(), bytes: 0 }; + row.keys.push(k); + if (!row.hashes.has(e.hash)) (row.hashes.add(e.hash), (row.bytes += e.bytes)); + byKind.set(e.kind, row); + } + for (const [kind, row] of byKind) + push({ + id: 'assetcache:' + kind, + category: 'assetcache', + label: labels[kind] ?? kind, + sub: row.hashes.size + (row.hashes.size === 1 ? ' file' : ' files'), + bytes: row.bytes, + removable: true, + kind: 'assetcache', + ref: row.keys + }); + } catch {} + /** @type {StorageCategory[]} */ const categories = CATEGORIES.map((c) => { const rows = (buckets.get(c.key) ?? []).sort((a, b) => b.bytes - a.bytes || a.label.localeCompare(b.label)); @@ -808,6 +840,9 @@ export async function reclaimRow(row) { case 'packcache': // 39 P5: the item stays in its pack; only this device's copy goes return await deletePackCache(/** @type {string[]} */ (row.ref ?? [])); + case 'assetcache': + // 60 Q6: only this device's copies go; the next use downloads (or converts) again + return await deleteCached(/** @type {string[]} */ (row.ref ?? [])); case 'packdims': await clearMeasuredDims(); return row.bytes; diff --git a/src/lib/vr/grip.js b/src/lib/vr/grip.js index a9b7fc95a..064b6311f 100644 --- a/src/lib/vr/grip.js +++ b/src/lib/vr/grip.js @@ -116,14 +116,18 @@ export function broadcastMove(object, force = false) { lastMoveSent = now; /** @type {any} */ const peer = get(peers); + // 60 Q2: the held stream is fast; a FORCED send (the release) is the pose that must land if (peer) - peer.send({ - type: 'move', - uuid: object.uuid, - pos: object.position.toArray(), - rot: object.rotation.toArray(), - scale: object.scale.toArray() - }); + peer.send( + { + type: 'move', + uuid: object.uuid, + pos: object.position.toArray(), + rot: object.rotation.toArray(), + scale: object.scale.toArray() + }, + { fast: !force } + ); } /** @param {any} object @param {any} before null = a PLAYER's grab (Interact): moved and diff --git a/tests/e2e/asset-cache-opfs.test.cjs b/tests/e2e/asset-cache-opfs.test.cjs new file mode 100644 index 000000000..3c67c4dfc --- /dev/null +++ b/tests/e2e/asset-cache-opfs.test.cjs @@ -0,0 +1,201 @@ +// 60 Q6 — THE LOCAL ASSET CACHE (OPFS): +// §1 a pack file and a sky are downloaded ONCE: after a reload the second load reads them from +// OPFS and the network sees no request for either (counted on the wire, not believed) +// §2 navigator.storage.persist() is asked exactly once, from the toast that says why — not on +// boot, not again after a reload +// §3 the app shell is never kept: only content keys are in the index +// §4 Settings ▸ Storage shows what it holds and Clear empties it (index and OPFS files) +// §5 39 P5's CacheStorage pack copies are moved over and that cache deleted +// +// APP_URL=https://localhost:5452/ node tests/e2e/asset-cache-opfs.test.cjs +const fs = require('fs'); +const path = require('path'); +const h = require('./helpers.cjs'); + +const STATIC = path.join(__dirname, '../../static/library'); +const DUCK = + 'https://cdn.jsdelivr.net/gh/theprototype-app/packs@format-1/default/Duck/glTF-Binary/Duck.glb'; +const DUCK_BYTES = fs.statSync(path.join(STATIC, 'default/Duck/glTF-Binary/Duck.glb')).size; +const SKY = 'hdri/meadow_2_1k.hdr'; +const LEGACY = + 'https://cdn.jsdelivr.net/gh/theprototype-app/packs@format-1/default/legacy-probe.glb'; + +/** counts persist() calls across reloads (sessionStorage survives a reload in one tab) and + * makes persisted() answer false, so the app has a reason to ask */ +function persistSpy() { + const storage = navigator.storage; + storage.persist = async () => { + sessionStorage.setItem( + 'e2e:persist', + String(Number(sessionStorage.getItem('e2e:persist') || 0) + 1) + ); + return true; + }; + storage.persisted = async () => Number(sessionStorage.getItem('e2e:persist') || 0) > 0; +} + +h.run(async () => { + const browser = await h.launch(); + const A = await h.setupPage(browser, 'A', { init: persistSpy }); + const hits = { duck: 0, sky: 0 }; + A.page.on('request', (req) => { + const u = req.url(); + if (/Duck\.glb$/.test(u)) hits.duck++; + if (u.endsWith('/' + SKY)) hits.sky++; + }); + await A.ctx.route(/cdn\.jsdelivr\.net\/gh\/theprototype-app\/packs@[^/]+\/.*\.glb$/, (route) => + route.fulfill({ + contentType: 'model/gltf-binary', + body: fs.readFileSync(path.join(STATIC, 'default/Duck/glTF-Binary/Duck.glb')) + }) + ); + + const facts = () => + A.page.evaluate(async () => { + const s = window.__stores; + await s.assetCache.assetCacheReady(); + let idx; + s.assetCache.assetCacheIndex.subscribe((v) => (idx = v))(); + return { + stats: s.assetCache.assetCacheStats(), + keys: Object.keys(idx.entries), + persist: Number(sessionStorage.getItem('e2e:persist') || 0) + }; + }); + + // ---- §5 first: a 39 P5 CacheStorage copy is waiting from an older build ---------------------- + await A.page.evaluate(async (url) => { + const c = await caches.open('tp-pack-files-v1'); + await c.put( + url, + new Response(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]), { + headers: { 'content-type': 'model/gltf-binary' } + }) + ); + }, LEGACY); + + let f = await facts(); + h.check(f.stats.backend === 'opfs', 'the cache runs on OPFS (' + f.stats.backend + ')'); + h.check(f.persist === 0, '§2 premise: persist() was not asked on boot'); + + // ---- §1 first load: downloads ----------------------------------------------------------------- + const first = await A.page.evaluate( + async ({ duck, sky }) => { + const s = window.__stores; + const a = await s.packCache.fetchPackBuffer(duck); + const b = await s.assetCache.fetchCached(new URL(sky, document.baseURI).href, { + kind: 'sky', + version: 'e2e' + }); + return { duck: a.byteLength, sky: b.byteLength }; + }, + { duck: DUCK, sky: SKY } + ); + h.check( + first.duck === DUCK_BYTES && hits.duck === 1, + `first load fetched the pack file once (${first.duck} bytes, ${hits.duck} request)` + ); + h.check( + first.sky > 100000 && hits.sky === 1, + `first load fetched the sky once (${first.sky} bytes, ${hits.sky} request)` + ); + await h.eventually(facts, (x) => x.keys.length >= 2, 'both are in the cache index'); + + // §5: the legacy copy moved over and the old cache is gone + await A.page.evaluate(() => window.__stores.packCache.loadPackCacheIndex()); + const legacy = await A.page.evaluate( + async (url) => ({ + old: await caches.has('tp-pack-files-v1'), + held: !!(await window.__stores.packCache.cachedPackBuffer(url)) + }), + LEGACY + ); + h.check( + !legacy.old && legacy.held, + '§5 the 39 P5 CacheStorage copy moved into the asset cache and that cache is deleted' + ); + + // ---- §2 the question, once ---------------------------------------------------------------------- + const toast = A.page.locator('.tp-toast', { hasText: 'kept on this device' }).first(); + await toast.waitFor({ timeout: 10000 }); + h.check(true, '§2 the toast says why it wants to keep the cache'); + await toast.getByRole('button', { name: 'Keep them' }).click(); + await h.eventually( + facts, + (x) => x.persist === 1, + '§2 "Keep them" asked navigator.storage.persist() once' + ); + + // ---- reload: the second load is served from OPFS --------------------------------------------- + await A.page.waitForTimeout(800); // the index flush (400 ms debounce) + await h.freshReload(A); + await h.answerRestore(A).catch(() => {}); + const second = await A.page.evaluate( + async ({ duck, sky }) => { + const s = window.__stores; + const a = await s.packCache.fetchPackBuffer(duck); + const b = await s.assetCache.fetchCached(new URL(sky, document.baseURI).href, { + kind: 'sky', + version: 'e2e' + }); + return { duck: a.byteLength, sky: b.byteLength, stats: s.assetCache.assetCacheStats() }; + }, + { duck: DUCK, sky: SKY } + ); + h.check( + second.duck === DUCK_BYTES && second.sky === first.sky, + '§1 the second load has the same bytes' + ); + h.check( + hits.duck === 1 && hits.sky === 1, + `§1 the second load made NO request: pack ${hits.duck}, sky ${hits.sky} (total since boot)` + ); + h.check( + second.stats.hits >= 2 && second.stats.fetches === 0, + `§1 both came from the cache (hits ${second.stats.hits}, fetches ${second.stats.fetches})` + ); + await A.page.waitForTimeout(1500); + f = await facts(); + h.check( + f.persist === 1, + '§2 persist() still asked exactly once after the reload (' + f.persist + ')' + ); + h.check( + (await A.page.locator('.tp-toast', { hasText: 'kept on this device' }).count()) === 0, + '§2 and the toast does not come back' + ); + + // ---- §3 content only ---------------------------------------------------------------------------- + const shell = f.keys.filter((k) => /\.(js|css|html)(#|$)|\/_app\/|sw\.js|version\.json/.test(k)); + h.check( + shell.length === 0, + '§3 nothing of the app shell is in the cache (' + JSON.stringify(f.keys) + ')' + ); + + // ---- §4 Settings ▸ Storage ----------------------------------------------------------------------- + await A.page.evaluate(() => { + window.__stores.settingsSection.set('storage'); + window.__stores.settingsOpen.set(true); + }); + const usage = A.page.locator('#asset-cache-usage'); + await usage.waitFor({ timeout: 15000 }); + const text = await usage.innerText(); + h.check( + /packs/.test(text) && /skies/.test(text) && /\d/.test(text), + '§4 Settings ▸ Storage shows the use by kind: "' + text.slice(0, 90) + '…"' + ); + await A.page.locator('#asset-cache-clear').click(); + await h.eventually(facts, (x) => x.keys.length === 0, '§4 Clear empties the index'); + const files = await A.page.evaluate(async () => { + const root = await navigator.storage.getDirectory(); + const dir = await root.getDirectoryHandle('tp-asset-cache'); + const blobs = await dir.getDirectoryHandle('blobs'); + let n = 0; + for await (const _ of blobs.keys()) n++; + return n; + }); + h.check(files === 0, '§4 and the OPFS blob files (' + files + ' left)'); + h.check(/Freed/.test(await usage.innerText()), '§4 it says what it freed'); + + await h.finish(browser); +}); diff --git a/tests/e2e/fast-channel-fallback.test.cjs b/tests/e2e/fast-channel-fallback.test.cjs new file mode 100644 index 000000000..e1c49d726 --- /dev/null +++ b/tests/e2e/fast-channel-fallback.test.cjs @@ -0,0 +1,81 @@ +// 60 Q2 — GRACEFUL FALLBACK: a peer without the fast channel (an older build; here B with +// `fastChannel = off` from boot, so it advertises no `fc` and ignores the hello) gets every +// pose down the reliable conn, and nothing is opened on either side. Then the same pair the +// other way round: B's own stream reaches A. And the avatar really moves on the receiver. +const h = require('./helpers.cjs'); + +h.run(async () => { + const browser = await h.launch(); + const A = await h.setupPage(browser, 'A'); + const B = await h.setupPage(browser, 'B', { storage: { fastChannel: 'off' } }); + await h.connect(A, B); + await A.page.waitForTimeout(2000); + + const open = { + a: await A.page.evaluate((id) => window.__stores.fastChannel.isOpen(id), B.id), + b: await B.page.evaluate((id) => window.__stores.fastChannel.isOpen(id), A.id), + aPeers: await A.page.evaluate(() => window.__stores.fastChannel.links.peers()) + }; + h.check( + !open.a && !open.b && open.aPeers.length === 0, + 'no fast channel is opened with a peer that lacks it ' + JSON.stringify(open) + ); + + /** stream n camera poses from `from` with {fast:true}; `to` records how they arrived */ + async function stream(from, to, n) { + await to.page.evaluate((fid) => { + window.__poses = []; + window.__stores.fastChannel.setPoseTap((peer, d, via) => { + if (peer === fid && d.type === 'camera') window.__poses.push({ via, x: d.position[0] }); + }); + }, from.id); + await from.page.evaluate( + (n) => + new Promise((resolve) => { + let p; + window.__stores.peers.subscribe((x) => (p = x))(); + let i = 0; + const t = setInterval(() => { + p.send( + { + type: 'camera', + peerId: p.peer.id, + position: [2 + i * 0.01, 1.6, -1], + rotation: [0, 0.5, 0] + }, + { fast: true } + ); + if (++i >= n) (clearInterval(t), resolve(null)); + }, 50); + }), + n + ); + await to.page.waitForTimeout(1500); + return to.page.evaluate(() => window.__poses); + } + + const toB = await stream(A, B, 60); + h.check(toB.length === 60, `A -> B: every pose arrived on the fallback (${toB.length}/60)`); + h.check( + toB.every((p) => p.via === 'reliable'), + 'A -> B: all of them on the reliable conn' + ); + const avatar = await B.page.evaluate((aid) => { + let scene; + window.__stores.globalScene.subscribe((s) => (scene = s))(); + const m = scene?.getObjectByName(aid); + return m ? m.position.x : null; + }, A.id); + h.check( + avatar !== null && Math.abs(avatar - (2 + 59 * 0.01)) < 1e-6, + `B draws A's avatar at the last pose (${avatar})` + ); + + const toA = await stream(B, A, 40); + h.check( + toA.length === 40 && toA.every((p) => p.via === 'reliable'), + `B -> A: every pose arrived, reliable (${toA.length}/40)` + ); + + await h.finish(browser); +}); diff --git a/tests/e2e/fast-channel-loss.test.cjs b/tests/e2e/fast-channel-loss.test.cjs new file mode 100644 index 000000000..72c0af380 --- /dev/null +++ b/tests/e2e/fast-channel-loss.test.cjs @@ -0,0 +1,334 @@ +// 60 Q2 — THE FAST MOVEMENT CHANNEL ON A LOSSY LINK, MEASURED. +// +// Two real peers. A streams its avatar pose at 20 Hz through the app's own send path +// (`peers.send({type:'camera'…}, {fast: true})`, what Scene.svelte does); B times every pose +// the app APPLIES (the `fastChannel.setPoseTap` seam sits after the stale filter, on both +// channels) and the gaps between them give the p95. Six phases on ONE connection: +// before (A's channel switched off = every pose on the reliable conn, the pre-Q2 wire) and +// after, each at 0 / 5 / 10 % loss. +// +// THE LOSS MODEL (a WebRTC loss shim — tc netem needs root and would hit every lane on this +// box). Installed on both pages before boot, it wraps every RTCDataChannel's message delivery: +// * every message gets a one-way delay of BASE_MS ± JITTER_MS; +// * on an UNRELIABLE channel (maxRetransmits set) a lost message is gone; +// * on a RELIABLE ordered channel a lost message is RETRANSMITTED RTX_MS later (doubling on a +// repeated loss, the SCTP back-off) and every later message waits behind it — head-of-line +// blocking, which is what SCTP's ordered delivery does with a gap. +// RTX_MS = 200 stands for SCTP's recovery on a 20 Hz stream: fast retransmit needs three later +// packets (~150 ms at 20 Hz) plus a round trip; the RTO path is slower still. A seeded PRNG per +// page, so a rerun draws the same losses. +// +// Also asserted: a reliable EDIT stream (numbered chat messages) arrives complete and in order +// in every phase; and the trailing keyframe lands A's final pose exactly after the stream stops. +// Evidence: EVIDENCE/fast-channel-loss.json + .md (table of p95 / p50 / max / received). +const fs = require('fs'); +const path = require('path'); +const h = require('./helpers.cjs'); + +const BASE_MS = 20; +const JITTER_MS = 5; +const RTX_MS = 200; +const HZ = 20; +const PHASE_MS = Number(process.env.FC_PHASE_MS || 20000); +const LOSSES = [0, 0.05, 0.1]; +const EVIDENCE = + process.env.EVIDENCE || path.join(process.env.HOME || '.', '.code/lanes-30/after-60/60-net'); + +/** runs in the page before any app code */ +function lossShim(cfg) { + window.__loss = { + p: 0, + base: cfg.base, + jitter: cfg.jitter, + rtx: cfg.rtx, + lost: { fast: 0, reliable: 0 }, + msgs: { fast: 0, reliable: 0 } + }; + let seed = cfg.seed >>> 0; + const rand = () => { + seed = (seed + 0x6d2b79f5) >>> 0; + let t = seed; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + const state = new WeakMap(); + /** wrap one listener so delivery follows the model */ + function wrap(dc, fn) { + return function (event) { + const L = window.__loss; + const unreliable = dc.maxRetransmits !== null || dc.maxPacketLifeTime !== null; + const kind = unreliable ? 'fast' : 'reliable'; + L.msgs[kind]++; + let at = performance.now() + L.base + (rand() * 2 - 1) * L.jitter; + if (L.p > 0 && rand() < L.p) { + L.lost[kind]++; + if (unreliable) return; // gone + let rtx = L.rtx; + at += rtx; + while (rand() < L.p) { + rtx *= 2; + at += rtx; + } + } + const self = this; + if (unreliable) { + setTimeout(() => fn.call(self, event), Math.max(0, at - performance.now())); + return; + } + // ordered: nothing overtakes the message before it. ONE FIFO per channel drained by ONE + // pump — a timer per message does not keep order: Chrome rounds each delay to whole ms, so + // two messages released by the same retransmit swapped (run4: reliable@10 % applied 201 of + // 400 poses through the stale filter, and the in-order edit check failed) + let s = state.get(dc); + if (!s) state.set(dc, (s = { last: 0, queue: [], timer: 0 })); + at = Math.max(at, s.last); + s.last = at; + s.queue.push({ at, run: () => fn.call(self, event) }); + const pump = () => { + s.timer = 0; + while (s.queue.length && s.queue[0].at <= performance.now()) { + try { + s.queue.shift().run(); + } catch (e) { + setTimeout(() => { + throw e; + }); + } + } + if (s.queue.length) + s.timer = setTimeout(pump, Math.max(0, s.queue[0].at - performance.now())); + }; + if (!s.timer) s.timer = setTimeout(pump, Math.max(0, s.queue[0].at - performance.now())); + }; + } + const proto = RTCDataChannel.prototype; + const desc = Object.getOwnPropertyDescriptor(proto, 'onmessage'); + Object.defineProperty(proto, 'onmessage', { + configurable: true, + get() { + return this.__onmsg ?? null; + }, + set(fn) { + this.__onmsg = fn; + desc.set.call(this, typeof fn === 'function' ? wrap(this, fn) : fn); + } + }); + const add = proto.addEventListener; + proto.addEventListener = function (type, fn, opts) { + if (type === 'message' && typeof fn === 'function') + return add.call(this, type, wrap(this, fn), opts); + return add.call(this, type, fn, opts); + }; +} + +/** @param {number[]} xs @param {number} q */ +const quantile = (xs, q) => { + if (!xs.length) return NaN; + const s = [...xs].sort((a, b) => a - b); + return s[Math.min(s.length - 1, Math.floor(q * (s.length - 1) + 0.5))]; +}; + +h.run(async () => { + const browser = await h.launch({ args: h.GPU_ARGS }); + const A = await h.setupPage(browser, 'A', { + init: [lossShim, { base: BASE_MS, jitter: JITTER_MS, rtx: RTX_MS, seed: 6001 }] + }); + const B = await h.setupPage(browser, 'B', { + init: [lossShim, { base: BASE_MS, jitter: JITTER_MS, rtx: RTX_MS, seed: 6002 }] + }); + await h.connect(A, B); + + // the channel opened both ways (the modules `fc` + `fastchan` hello) + await h.eventually( + async () => ({ + a: await A.page.evaluate((id) => window.__stores.fastChannel.isOpen(id), B.id), + b: await B.page.evaluate((id) => window.__stores.fastChannel.isOpen(id), A.id) + }), + (r) => r.a && r.b, + 'the fast channel is open on both ends after the handshake' + ); + const hasPc = await A.page.evaluate((id) => { + let p; + window.__stores.peers.subscribe((x) => (p = x))(); + return !!p.connections[id]?.peerConnection; + }, B.id); + h.check(hasPc, 'the PeerJS connection exposes its RTCPeerConnection'); + + // A headless peer renders at a few fps and every frame blocks its main thread for hundreds of + // ms — a 20 Hz timer then fires at ~1 Hz and every gap measures the RENDERER, not the wire (run2: + // 17 sends in 20 s, p50 1.2 s). Both pages stop drawing through the app's own overload pause + // for the measurement; the data channels, the stale filter and the keyframe timer do not draw. + for (const peer of [A, B]) + await peer.page.evaluate(() => + window.__stores.overloadGuard.pauseRendering('e2e: timing the wire') + ); + + // B's tap: arrival times of A's applied camera poses, and the chat edits in arrival order + await B.page.evaluate((aid) => { + window.__poses = []; + window.__stores.fastChannel.setPoseTap((peer, d, via) => { + if (peer === aid && d.type === 'camera') + window.__poses.push({ t: performance.now(), via, x: d.position[0], ps: d.ps }); + }); + }, A.id); + + /** one phase: set loss on both, stream for PHASE_MS, collect B's gaps */ + async function phase(mode, loss) { + const fastOn = mode === 'after'; + for (const peer of [A, B]) await peer.page.evaluate((p) => (window.__loss.p = p), loss); + await A.page.evaluate( + (on) => + on ? localStorage.removeItem('fastChannel') : localStorage.setItem('fastChannel', 'off'), + fastOn + ); + await B.page.evaluate(() => (window.__poses = [])); + const tag = mode + '-' + Math.round(loss * 100); + const sent = await A.page.evaluate( + ({ ms, hz, tag }) => + new Promise((resolve) => { + let p; + window.__stores.peers.subscribe((x) => (p = x))(); + let n = 0; + let edits = 0; + const t0 = performance.now(); + const timer = setInterval(() => { + const x = n * 0.01; + // exactly the call Scene.svelte makes for a moving camera + p.send( + { type: 'camera', peerId: p.peer.id, position: [x, 1.6, 0], rotation: [0, 0, 0] }, + { fast: true } + ); + n++; + // a reliable EDIT every 10th tick (2 a second), numbered + if (n % 10 === 0) p.sendMessage('edit ' + tag + ' ' + edits++); + if (performance.now() - t0 >= ms) { + clearInterval(timer); + resolve({ n, edits, lastX: (n - 1) * 0.01 }); + } + }, 1000 / hz); + }), + { ms: PHASE_MS, hz: HZ, tag } + ); + // let retransmits drain and the trailing keyframe land: at 10 % the doubling back-off can stack past + // a second (run3: 1.3 s max gap), so wait for the last edit AND the final pose, not a fixed time — + // a late message would otherwise land in the NEXT phase's samples + await B.page + .waitForFunction( + ({ tag, edits, lastX }) => { + let msgs = []; + window.__stores.messages?.subscribe?.((m) => (msgs = m))(); + const n = (msgs || []).filter( + (m) => m?.type === 'received' && String(m?.text ?? '').startsWith('edit ' + tag + ' ') + ).length; + const last = window.__poses[window.__poses.length - 1]; + return n >= edits && !!last && Math.abs(last.x - lastX) < 1e-9; + }, + { tag, edits: sent.edits, lastX: sent.lastX }, + { timeout: 10000 } + ) + .catch(() => {}); + await B.page.waitForTimeout(300); + const got = await B.page.evaluate((tag) => { + let msgs = []; + window.__stores.messages?.subscribe?.((m) => (msgs = m))(); + const edits = (msgs || []) + .filter((m) => m?.type === 'received') + .map((m) => String(m?.text ?? '')) + .filter((m) => m.startsWith('edit ' + tag + ' ')) + .map((m) => Number(m.split(' ').pop())); + return { poses: window.__poses, edits, loss: window.__loss }; + }, tag); + const times = got.poses.map((p) => p.t); + const gaps = times.slice(1).map((t, i) => t - times[i]); + const viaFast = got.poses.filter((p) => p.via === 'fast').length; + const lastX = got.poses.length ? got.poses[got.poses.length - 1].x : NaN; + const row = { + mode, + loss, + sent: sent.n, + received: got.poses.length, + viaFast, + p50: Math.round(quantile(gaps, 0.5)), + p95: Math.round(quantile(gaps, 0.95)), + p99: Math.round(quantile(gaps, 0.99)), + max: Math.round(Math.max(...gaps)), + editsSent: sent.edits, + editsInOrder: got.edits.length === sent.edits && got.edits.every((v, i) => v === i), + finalPoseLanded: Math.abs(lastX - sent.lastX) < 1e-9 + }; + console.log(JSON.stringify(row)); + // a phase whose sender could not keep the cadence measured the page, not the channel + h.check( + sent.n >= (PHASE_MS / 1000) * HZ * 0.8, + `${tag}: A kept the ${HZ} Hz cadence (${sent.n} sends in ${PHASE_MS / 1000} s)` + ); + return row; + } + + const rows = []; + for (const loss of LOSSES) { + rows.push(await phase('before', loss)); + rows.push(await phase('after', loss)); + } + for (const peer of [A, B]) await peer.page.evaluate(() => (window.__loss.p = 0)); + + for (const r of rows) { + const name = `${r.mode} @ ${Math.round(r.loss * 100)} %`; + h.check(r.editsInOrder, `${name}: every reliable edit arrived, in order (${r.editsSent})`); + h.check(r.finalPoseLanded, `${name}: the final pose landed exactly (trailing keyframe)`); + if (r.mode === 'after') + h.check( + r.viaFast > r.received * 0.8, + `${name}: poses rode the fast channel (${r.viaFast}/${r.received})` + ); + else { + h.check(r.viaFast === 0, `${name}: the channel switched off sends every pose reliable`); + // the shim's own control: an ORDERED channel never trips the stale filter + h.check( + r.received >= r.sent, + `${name}: the ordered channel applied every pose (${r.received}/${r.sent})` + ); + } + } + const at = (mode, loss) => rows.find((r) => r.mode === mode && r.loss === loss); + h.check( + at('after', 0.05).p95 < 150, + `after @ 5 %: p95 pose gap < 150 ms (release bar) — ${at('after', 0.05).p95} ms` + ); + h.check( + at('after', 0.05).p95 < at('before', 0.05).p95, + `5 %: fast p95 ${at('after', 0.05).p95} < reliable p95 ${at('before', 0.05).p95}` + ); + h.check( + at('after', 0.1).p95 < at('before', 0.1).p95, + `10 %: fast p95 ${at('after', 0.1).p95} < reliable p95 ${at('before', 0.1).p95}` + ); + + try { + fs.mkdirSync(EVIDENCE, { recursive: true }); + const meta = { + at: new Date().toISOString(), + model: { BASE_MS, JITTER_MS, RTX_MS, HZ, PHASE_MS }, + rows + }; + fs.writeFileSync(path.join(EVIDENCE, 'fast-channel-loss.json'), JSON.stringify(meta, null, 2)); + const md = [ + '# 60 Q2 — pose gap on a lossy link (fast-channel-loss e2e)', + '', + `Model: one-way ${BASE_MS} ± ${JITTER_MS} ms; reliable loss = retransmit after ${RTX_MS} ms (doubling) with head-of-line blocking; fast loss = dropped. ${HZ} Hz stream, ${PHASE_MS / 1000} s per phase, gaps between APPLIED poses at the receiver.`, + '', + '| channel | loss | sent | applied | p50 ms | p95 ms | p99 ms | max ms | edits in order | final pose |', + '|---|---|---|---|---|---|---|---|---|---|', + ...rows.map( + (r) => + `| ${r.mode === 'after' ? 'fast (after)' : 'reliable (before)'} | ${Math.round(r.loss * 100)} % | ${r.sent} | ${r.received} | ${r.p50} | ${r.p95} | ${r.p99} | ${r.max} | ${r.editsInOrder ? 'yes' : 'NO'} | ${r.finalPoseLanded ? 'yes' : 'NO'} |` + ) + ]; + fs.writeFileSync(path.join(EVIDENCE, 'fast-channel-loss.md'), md.join('\n') + '\n'); + } catch (e) { + console.log('evidence write failed: ' + e); + } + + await h.finish(browser); +}); diff --git a/tests/e2e/place-drag-cache.test.cjs b/tests/e2e/place-drag-cache.test.cjs index 5198247c4..4cea135b9 100644 --- a/tests/e2e/place-drag-cache.test.cjs +++ b/tests/e2e/place-drag-cache.test.cjs @@ -59,7 +59,8 @@ const cacheFacts = (page) => s.packCache.packCacheIndex.subscribe((v) => (index = v))(); const urls = [...index.keys()]; const duck = urls.find((u) => /Duck\.glb$/.test(u)) ?? null; - const match = duck ? await (await caches.open(s.packCache.PACK_CACHE_NAME)).match(duck) : null; + // 60 Q6: the pack cache lives in the local asset cache (OPFS) now — ask it, not CacheStorage + const match = duck ? await s.packCache.cachedPackBuffer(duck) : null; return { urls, duck, bytes: duck ? index.get(duck) : 0, inCache: !!match, wanted: url }; }, DUCK_URL); diff --git a/tests/unit/assetCacheCore.test.js b/tests/unit/assetCacheCore.test.js new file mode 100644 index 000000000..41df5fda8 --- /dev/null +++ b/tests/unit/assetCacheCore.test.js @@ -0,0 +1,182 @@ +// @ts-nocheck — fixture indexes as plain literals; the module itself is typed +import { describe, it, expect } from 'vitest'; +import { + stableStringify, + urlKey, + urlOfKey, + derivedKey, + emptyIndex, + normalizeIndex, + blobsOf, + usageOf, + planEviction, + orphanedBy, + DEFAULT_CAP_MB, + CAP_CHOICES_MB +} from '../../src/lib/assetCacheCore.js'; + +// 60 Q6: the local asset cache's pure half — cache keys, the index, the LRU plan. + +const H = (c) => c.repeat(64); // a sha-256-shaped hex +const entry = (hash, bytes, lastUsed, kind = 'pack') => ({ hash, bytes, kind, lastUsed, at: 0 }); + +describe('cache keys', () => { + it('url keys: absolute http(s) only, fragment dropped, optional version tag', () => { + expect(urlKey('https://cdn.example/p/a.glb')).toBe('url:https://cdn.example/p/a.glb'); + expect(urlKey('https://cdn.example/p/a.glb#frag')).toBe('url:https://cdn.example/p/a.glb'); + expect(urlKey('https://x.test/hdri/sky.hdr', '1.41.0')).toBe( + 'url:https://x.test/hdri/sky.hdr#v=1.41.0' + ); + expect(urlKey('blob:https://x/1')).toBeNull(); + expect(urlKey('data:,x')).toBeNull(); + expect(urlKey('hdri/sky.hdr')).toBeNull(); + expect(urlKey(undefined)).toBeNull(); + }); + it('urlOfKey inverts a url key (without its version)', () => { + expect(urlOfKey(urlKey('https://a/b.glb'))).toBe('https://a/b.glb'); + expect(urlOfKey(urlKey('https://a/b.hdr', '9'))).toBe('https://a/b.hdr'); + expect(urlOfKey('derived:texture:abc:{}')).toBeNull(); + }); + it('derived keys: content hash + kind + settings, field order irrelevant', () => { + const a = derivedKey('texture', H('a'), { maxTextureSize: 2048, textures: 'ktx2+fallback' }); + const b = derivedKey('texture', H('A'), { textures: 'ktx2+fallback', maxTextureSize: 2048 }); + expect(a).toBe(b); + expect(a.startsWith('derived:texture:' + H('a') + ':')).toBe(true); + expect( + derivedKey('texture', H('a'), { maxTextureSize: 1024, textures: 'ktx2+fallback' }) + ).not.toBe(a); + expect( + derivedKey('texture', H('b'), { maxTextureSize: 2048, textures: 'ktx2+fallback' }) + ).not.toBe(a); + expect(derivedKey('texture', 'not-a-hash', {})).toBeNull(); + expect(derivedKey('', H('a'), {})).toBeNull(); + }); + it('stableStringify sorts keys at every depth and drops undefined fields', () => { + expect(stableStringify({ b: 1, a: { d: [2, { z: 1, y: 2 }], c: undefined } })).toBe( + '{"a":{"d":[2,{"y":2,"z":1}]},"b":1}' + ); + expect(stableStringify(null)).toBe('null'); + expect(stableStringify(undefined)).toBe('null'); + }); +}); + +describe('normalizeIndex', () => { + it('keeps good rows, drops malformed ones and unknown versions', () => { + const raw = { + v: 1, + entries: { + ok: { + hash: H('a'), + bytes: 10, + kind: 'sky', + lastUsed: 5, + at: 1, + type: 'image/vnd.radiance', + meta: { r: 1 } + }, + badHash: { hash: 'zz', bytes: 1, kind: 'pack' }, + badBytes: { hash: H('b'), bytes: -1, kind: 'pack' }, + oddKind: { hash: H('c'), bytes: 3, kind: 'weird' } + } + }; + const idx = normalizeIndex(raw); + expect(Object.keys(idx.entries).sort()).toEqual(['oddKind', 'ok']); + expect(idx.entries.ok).toEqual({ + hash: H('a'), + bytes: 10, + kind: 'sky', + type: 'image/vnd.radiance', + lastUsed: 5, + at: 1, + meta: { r: 1 } + }); + expect(idx.entries.oddKind.kind).toBe('import'); + expect(normalizeIndex({ v: 2, entries: raw.entries })).toEqual(emptyIndex()); + expect(normalizeIndex(null)).toEqual(emptyIndex()); + expect(normalizeIndex('junk')).toEqual(emptyIndex()); + }); +}); + +describe('usage', () => { + it('counts a blob shared by two keys ONCE', () => { + const idx = { + v: 1, + entries: { + k1: entry(H('a'), 100, 1), + k2: entry(H('a'), 100, 2, 'import'), + k3: entry(H('b'), 50, 3, 'sky') + } + }; + expect(blobsOf(idx).get(H('a'))).toEqual({ bytes: 100, refs: 2 }); + const u = usageOf(idx); + expect(u).toEqual({ + bytes: 150, + keys: 3, + blobs: 2, + byKind: { pack: 100, sky: 50, texture: 0, import: 0 } + }); + }); +}); + +describe('planEviction (LRU)', () => { + const idx = () => ({ + v: 1, + entries: { + old: entry(H('a'), 400, 10), + mid: entry(H('b'), 300, 20), + fresh: entry(H('c'), 300, 30) + } + }); + it('does nothing under the cap', () => { + expect(planEviction(idx(), 1000)).toEqual({ keys: [], hashes: [], bytesAfter: 1000 }); + }); + it('drops the least recently used first until it fits', () => { + expect(planEviction(idx(), 700)).toEqual({ keys: ['old'], hashes: [H('a')], bytesAfter: 600 }); + expect(planEviction(idx(), 350)).toEqual({ + keys: ['old', 'mid'], + hashes: [H('a'), H('b')], + bytesAfter: 300 + }); + }); + it('never drops a protected key (the one just written), even when it alone is over the cap', () => { + expect(planEviction(idx(), 100, ['fresh'])).toEqual({ + keys: ['old', 'mid'], + hashes: [H('a'), H('b')], + bytesAfter: 300 + }); + }); + it('a shared blob is freed only when its LAST key goes', () => { + const shared = { + v: 1, + entries: { a1: entry(H('a'), 500, 1), a2: entry(H('a'), 500, 50), b: entry(H('b'), 100, 10) } + }; + // a1 goes first but frees nothing (a2 still names the blob); b goes next and frees 100 + expect(planEviction(shared, 550)).toEqual({ + keys: ['a1', 'b'], + hashes: [H('b')], + bytesAfter: 500 + }); + }); + it('is deterministic on equal timestamps (key order)', () => { + const tie = { v: 1, entries: { z: entry(H('a'), 10, 1), y: entry(H('b'), 10, 1) } }; + expect(planEviction(tie, 10).keys).toEqual(['y']); + }); +}); + +describe('orphanedBy', () => { + it('names only the blobs no remaining key refers to', () => { + const idx = { + v: 1, + entries: { a1: entry(H('a'), 5, 1), a2: entry(H('a'), 5, 1), b: entry(H('b'), 5, 1) } + }; + expect(orphanedBy(idx, ['a1'])).toEqual([]); + expect(orphanedBy(idx, ['a1', 'a2', 'b']).sort()).toEqual([H('a'), H('b')]); + expect(orphanedBy(idx, ['missing'])).toEqual([]); + }); +}); + +describe('the cap choices', () => { + it('offer the default', () => { + expect(CAP_CHOICES_MB).toContain(DEFAULT_CAP_MB); + }); +}); diff --git a/tests/unit/fastChannel.test.js b/tests/unit/fastChannel.test.js new file mode 100644 index 000000000..8c7146cb3 --- /dev/null +++ b/tests/unit/fastChannel.test.js @@ -0,0 +1,242 @@ +// @ts-nocheck — fake RTCPeerConnection / timer fixtures; the module itself is typed +import { describe, it, expect } from 'vitest'; +import { + poseKey, + stampPose, + nextPoseSeq, + createStaleFilter, + createKeyframer, + createFastLinks, + FAST_CHANNEL_ID, + FAST_MAX_BYTES, + FAST_MAX_BUFFERED +} from '../../src/lib/fastChannel.js'; + +// 60 Q2: the fast movement channel's pure parts — keys, sequence numbers, the stale filter, +// the trailing keyframe and the channel bookkeeping (with a fake RTCPeerConnection). + +describe('poseKey', () => { + it('keys camera/hands per sender stream and moves per object', () => { + expect(poseKey({ type: 'camera', peerId: 'a' })).toBe('camera'); + expect(poseKey({ type: 'vrhands' })).toBe('vrhands'); + expect(poseKey({ type: 'move', uuid: 'u1' })).toBe('move:u1'); + }); + it('answers null for everything that is not a latest-wins pose', () => { + expect(poseKey({ type: 'object' })).toBeNull(); + expect(poseKey({ type: 'move' })).toBeNull(); + expect(poseKey(null)).toBeNull(); + }); +}); + +describe('stampPose', () => { + it('stamps poses with a monotonic ps, once', () => { + const a = stampPose({ type: 'move', uuid: 'x' }); + const b = stampPose({ type: 'camera' }); + expect(b.ps).toBeGreaterThan(a.ps); + const keep = a.ps; + stampPose(a); // a keyframe re-send keeps its original number + expect(a.ps).toBe(keep); + }); + it('leaves non-pose payloads untouched', () => { + const edit = { type: 'object', uuid: 'x' }; + stampPose(edit); + expect('ps' in edit).toBe(false); + }); + it('starts above the clock x100 so a reloaded sender outruns its previous page', () => { + expect(nextPoseSeq()).toBeGreaterThan((Date.now() - 60_000) * 100); + }); +}); + +describe('createStaleFilter', () => { + it('drops a pose older than (or equal to) the newest applied for that sender + key', () => { + const f = createStaleFilter(); + expect(f.accept('A', { type: 'camera', ps: 10 })).toBe(true); + expect(f.accept('A', { type: 'camera', ps: 9 })).toBe(false); // overtaken datagram + expect(f.accept('A', { type: 'camera', ps: 10 })).toBe(false); // keyframe duplicate + expect(f.accept('A', { type: 'camera', ps: 11 })).toBe(true); + expect(f.stats).toEqual({ accepted: 2, stale: 2 }); + }); + it('keeps keys apart: per object, per sender, per type', () => { + const f = createStaleFilter(); + expect(f.accept('A', { type: 'move', uuid: 'u1', ps: 50 })).toBe(true); + expect(f.accept('A', { type: 'move', uuid: 'u2', ps: 5 })).toBe(true); + expect(f.accept('B', { type: 'move', uuid: 'u1', ps: 5 })).toBe(true); + expect(f.accept('A', { type: 'camera', ps: 5 })).toBe(true); + }); + it('a late streamed move loses to the committed move that ended the drag', () => { + const f = createStaleFilter(); + const streamed = { type: 'move', uuid: 'u', ps: 100 }; + const committed = { type: 'move', uuid: 'u', ps: 101 }; + expect(f.accept('A', committed)).toBe(true); + expect(f.accept('A', streamed)).toBe(false); + }); + it('always accepts an unstamped message (an older build) and records nothing', () => { + const f = createStaleFilter(); + expect(f.accept('A', { type: 'move', uuid: 'u' })).toBe(true); + expect(f.accept('A', { type: 'move', uuid: 'u' })).toBe(true); + expect(f.size()).toBe(0); + expect(f.accept('A', { type: 'object', ps: 1 })).toBe(true); // not a pose + }); + it('forgets a peer on disconnect (a fresh page may restart lower)', () => { + const f = createStaleFilter(); + f.accept('A', { type: 'camera', ps: 100 }); + f.accept('B', { type: 'camera', ps: 100 }); + f.dropPeer('A'); + expect(f.accept('A', { type: 'camera', ps: 1 })).toBe(true); + expect(f.accept('B', { type: 'camera', ps: 1 })).toBe(false); + }); +}); + +/** a manual clock for the keyframer */ +function fakeTimers() { + let now = 0; + const timers = new Map(); + let id = 0; + return { + setTimer: (fn, ms) => (timers.set(++id, { fn, at: now + ms }), id), + clearTimer: (t) => timers.delete(t), + advance(ms) { + now += ms; + for (const [t, v] of [...timers]) if (v.at <= now) (timers.delete(t), v.fn()); + }, + pending: () => timers.size + }; +} + +describe('createKeyframer', () => { + it('re-sends the LAST fast pose once the key goes quiet, to the peers that got it fast', () => { + const t = fakeTimers(); + const sent = []; + const k = createKeyframer((p, peers) => sent.push([p.ps, peers]), { delay: 250, ...t }); + k.noteFast('camera', { ps: 1 }, ['B']); + t.advance(100); + k.noteFast('camera', { ps: 2 }, ['B', 'C']); + t.advance(200); + expect(sent).toEqual([]); // still streaming + t.advance(60); + expect(sent).toEqual([[2, ['B', 'C']]]); + t.advance(1000); + expect(sent.length).toBe(1); // once + }); + it('a reliable send of the same key supersedes the keyframe', () => { + const t = fakeTimers(); + const sent = []; + const k = createKeyframer((p) => sent.push(p), { delay: 250, ...t }); + k.noteFast('move:u', { ps: 1 }, ['B']); + k.noteReliable('move:u'); + t.advance(1000); + expect(sent).toEqual([]); + expect(t.pending()).toBe(0); + }); + it('keys are independent; a departed peer is owed nothing; clear disarms all', () => { + const t = fakeTimers(); + const sent = []; + const k = createKeyframer((p, peers) => sent.push([p.k, peers]), { delay: 10, ...t }); + k.noteFast('move:a', { k: 'a' }, ['B', 'C']); + k.noteFast('move:b', { k: 'b' }, ['B']); + k.dropPeer('C'); + t.advance(20); + expect(sent).toEqual([ + ['a', ['B']], + ['b', ['B']] + ]); + k.noteFast('move:c', { k: 'c' }, ['B']); + k.clear(); + t.advance(20); + expect(sent.length).toBe(2); + }); +}); + +/** a fake RTCPeerConnection whose channels open on demand */ +function fakePc() { + const made = []; + return { + made, + createDataChannel(label, init) { + const dc = { + label, + init, + readyState: 'connecting', + sent: [], + send(t) { + this.sent.push(t); + }, + close() { + this.readyState = 'closed'; + this.onclose?.(); + } + }; + made.push(dc); + return dc; + } + }; +} + +describe('createFastLinks', () => { + it('creates ONE negotiated unordered, no-retransmit channel per RTCPeerConnection', () => { + const links = createFastLinks(); + const pc = fakePc(); + const conn = { peerConnection: pc }; + expect(links.open('B', conn, () => {})).toBe(true); + expect(links.open('B', conn, () => {})).toBe(true); // idempotent + expect(pc.made.length).toBe(1); + expect(pc.made[0].init).toEqual({ + ordered: false, + maxRetransmits: 0, + negotiated: true, + id: FAST_CHANNEL_ID + }); + }); + it('falls back (false) until the channel is open, and for an oversized message', () => { + const links = createFastLinks(); + const pc = fakePc(); + links.open('B', { peerConnection: pc }, () => {}); + expect(links.trySend('B', '{}')).toBe(false); // connecting + pc.made[0].readyState = 'open'; + pc.made[0].onopen?.(); + expect(links.trySend('B', '{"type":"camera"}')).toBe(true); + expect(links.trySend('B', 'x'.repeat(FAST_MAX_BYTES + 1))).toBe(false); + expect(links.trySend('Z', '{}')).toBe(false); // no channel to that peer + expect(links.stats.tooBig).toBe(1); + }); + it('skips (true, not sent) behind a send backlog — a stale pose must not queue', () => { + const links = createFastLinks(); + const pc = fakePc(); + links.open('B', { peerConnection: pc }, () => {}); + pc.made[0].readyState = 'open'; + pc.made[0].bufferedAmount = FAST_MAX_BUFFERED + 1; + expect(links.trySend('B', '{}')).toBe(true); + expect(pc.made[0].sent).toEqual([]); + expect(links.stats.congested).toBe(1); + }); + it('a newer connection replaces the old channel; a conn without an RTCPeerConnection is refused', () => { + const links = createFastLinks(); + const pc1 = fakePc(); + const pc2 = fakePc(); + links.open('B', { peerConnection: pc1 }, () => {}); + links.open('B', { peerConnection: pc2 }, () => {}); + expect(pc1.made[0].readyState).toBe('closed'); + expect(links.peers()).toEqual(['B']); + expect(links.open('C', {}, () => {})).toBe(false); + }); + it('parses JSON text into the receive path and ignores junk', () => { + const links = createFastLinks(); + const pc = fakePc(); + const got = []; + links.open('B', { peerConnection: pc }, (d) => got.push(d)); + pc.made[0].onmessage({ data: '{"type":"move","uuid":"u","ps":3}' }); + pc.made[0].onmessage({ data: 'not json' }); + pc.made[0].onmessage({ data: new ArrayBuffer(2) }); + expect(got).toEqual([{ type: 'move', uuid: 'u', ps: 3 }]); + expect(links.stats.parseErrors).toBe(1); + }); + it('close / closeAll drop the links', () => { + const links = createFastLinks(); + links.open('B', { peerConnection: fakePc() }, () => {}); + links.open('C', { peerConnection: fakePc() }, () => {}); + links.close('B'); + expect(links.peers()).toEqual(['C']); + links.closeAll(); + expect(links.peers()).toEqual([]); + }); +});