From cc1114772caf73de3070e0adc5367b4df7a5a17e Mon Sep 17 00:00:00 2001 From: Christopher Toth Date: Sun, 4 Oct 2026 22:47:43 -0600 Subject: [PATCH 1/2] Keep a sound's reverb send level as it moves away Cacophony takes a send from a voice's output, after its gain (routableSource.ts: outputNode.connect(sendGain), where outputNode is the gain node that follows the panner). The client applies a positioned sound's distance attenuation at that same gain, so the level reaching the room's reverb fell exactly as fast as the direct level. A far sound was quieter but no wetter than a near one, and the direct-to-reverb ratio, the main cue for distance other than loudness, never changed. The send is now given the gain send / distanceGain, so the chain receives volume x send wherever the source is and the direct level alone falls with distance. It is re-gained in place whenever the distance gain changes (the listener or the source moves, the spatial profile changes) and whenever the send or the chain changes. The feed from a sound's inline effect chain into its named chain is downstream of the same gain and is compensated the same way. The boost is capped at 32 x (+30 dB). The engine accepts any finite send gain and does not clamp it, and the distance curves give no usable limit: the default one floors at 1 / 10 000 and a linear one reaches 0, where the tap is silent. Under the default curve the reverberant level is even out to 32 m and falls with the direct sound beyond. A change of send on a live aux send now re-gains it in the engine instead of removing the send and adding it again. The direct level, the distance model and occlusion are unchanged: occlusion sits before the tap and still dims direct and reverb alike. Non-positional sounds have no distance gain and keep their wire send. Ambisonic sounds do not use named chains. --- src/audio/MediaService.ts | 64 +++++++++- src/audio/distanceModel.test.ts | 29 +++++ src/audio/distanceModel.ts | 23 ++++ src/audio/effects/EffectChain.ts | 7 ++ src/audio/namedRoute.ts | 28 ++++- src/gmcp/Client/Media.test.ts | 205 ++++++++++++++++++++++++++++++- 6 files changed, 346 insertions(+), 10 deletions(-) diff --git a/src/audio/MediaService.ts b/src/audio/MediaService.ts index b537f152..9bb91a5a 100644 --- a/src/audio/MediaService.ts +++ b/src/audio/MediaService.ts @@ -21,6 +21,7 @@ import type { PositionalFoaRenderer } from './PositionalFoaRenderer'; import { DEFAULT_SPATIAL_PROFILE, distanceBetween, + distanceCompensatedSend, profileDistanceGain, type SpatialProfile, } from './distanceModel'; @@ -385,11 +386,24 @@ export class MediaService { sound.ambisonicRenderer.setDistanceGain(this.distanceGain(sound)); } + /** + * The distance gain that is part of the sound's own gain: its profile's, for + * an HRTF point source, and 1 for every other sound (non-positional sounds + * have none; ambisonic routes apply distance in their renderer instead). + */ + private levelDistanceGain(sound: ExtendedSound): number { + return sound.pointSource && sound.upmix !== 'ambisonic' ? this.distanceGain(sound) : 1; + } + /** * Set the sound's gain: wire volume × 10^(gainDb/20) × (for an HRTF point * source) its profile distance gain. This is the one stage where distance * falloff is applied on the HRTF route; the panner's rolloff is 0. Ambisonic * routes apply distance in their renderer instead. + * + * The engine taps a send after this gain, so the send to the sound's chain + * is re-gained here too: every change of the distance gain (the listener or + * the source moves, the profile changes) comes through this method. */ private applyLevels(sound: ExtendedSound): void { if (this.cleanedSounds.has(sound)) { @@ -397,8 +411,40 @@ export class MediaService { } const volume = sound.mediaVolume ?? 1; const gain = sound.gainDb ? 10 ** (sound.gainDb / 20) : 1; - const distance = sound.pointSource && sound.upmix !== 'ambisonic' ? this.distanceGain(sound) : 1; - sound.volume = volume * gain * distance; + sound.volume = volume * gain * this.levelDistanceGain(sound); + this.refreshSend(sound); + } + + /** + * The gain for the sound's send to its named chain. Cacophony takes a send + * from the voice's output, after its gain, so the wire `send` alone would + * make the reverberant level fall with distance exactly as the direct level + * does. A room's reverberant field is roughly even, and the direct-to-reverb + * ratio is how a listener hears distance: so the distance gain is divided + * back out of the send ({@link distanceCompensatedSend}) and the chain gets + * volume × send wherever the source is. Occlusion sits before the tap and + * still dims direct and reverb alike. + */ + private sendGain(sound: ExtendedSound, send: number | undefined): number | undefined { + return send === undefined + ? undefined + : distanceCompensatedSend(send, this.levelDistanceGain(sound)); + } + + /** + * Bring a live send up to date with the sound's distance gain. Only a send + * that is already routed is touched: one the engine refused is not retried + * on every listener step. + */ + private refreshSend(sound: ExtendedSound): void { + if (sound.namedSend === undefined) { + return; + } + if (sound.effectChain) { + this.pointInlineChain(sound.effectChain, sound.namedChain, this.sendGain(sound, sound.namedSend)); + } else if (sound.chainRouted) { + this.routeNamedChain(sound, sound.namedChain, sound.namedSend); + } } /** HRTF panner settings for a point source: position and cone only, no native rolloff. */ @@ -1130,7 +1176,13 @@ export class MediaService { * undoing a different named route. A no-op when that route is already live. */ private routeNamedChain(sound: ExtendedSound, chain: string | undefined, send: number | undefined): void { - const error = routeNamedChain(sound, chain, send, this.cacophony.getBus('master')); + const error = routeNamedChain( + sound, + chain, + send, + this.cacophony.getBus('master'), + this.sendGain(sound, send), + ); if (error) { console.warn(`Client.Media: chain '${chain}' unavailable; playing dry`, error); this.traceSound('routed', sound, sound.key, { @@ -1195,7 +1247,7 @@ export class MediaService { // Update without effects: keep the inline chain, re-point what it feeds. sound.namedChain = chain; sound.namedSend = send; - this.pointInlineChain(sound.effectChain, chain, send); + this.pointInlineChain(sound.effectChain, chain, this.sendGain(sound, send)); return; } this.routeNamedChain(sound, chain, send); @@ -1237,6 +1289,8 @@ export class MediaService { * inline output stays on master (the dry path) and feeds the named chain at * the send level, exactly as chain+send behaves without inline effects. * Without a send, the inline chain runs in series into the named chain. + * `send` is the gain for that feed ({@link sendGain}): the inline bus is + * downstream of the sound's gain, so its feed is after the distance gain too. */ private pointInlineChain(inline: EffectChain, chain: string | undefined, send: number | undefined): void { const target = chain ? (this.effects.getChain(chain)?.bus ?? null) : null; @@ -1258,7 +1312,7 @@ export class MediaService { if (!inline) { return; } - this.pointInlineChain(inline, data.chain, data.send); + this.pointInlineChain(inline, data.chain, this.sendGain(sound, data.send)); try { sound.routeTo(inline.bus); } catch (error) { diff --git a/src/audio/distanceModel.test.ts b/src/audio/distanceModel.test.ts index 488b5037..c84440f3 100644 --- a/src/audio/distanceModel.test.ts +++ b/src/audio/distanceModel.test.ts @@ -3,7 +3,9 @@ import { describe, expect, it } from 'vitest'; import { DEFAULT_SPATIAL_PROFILE, distanceBetween, + distanceCompensatedSend, inverseDistanceGain, + MAX_SEND_DISTANCE_BOOST, profileDistanceGain, SPATIAL_DISTANCE_MODEL, type SpatialProfile, @@ -100,3 +102,30 @@ describe('distanceBetween', () => { expect(distanceBetween([1, 2], [1, 2, 3])).toBe(0); }); }); + +describe('distanceCompensatedSend', () => { + it('undoes the distance gain, so gain x send is the requested send', () => { + expect(distanceCompensatedSend(0.3, 0.25)).toBeCloseTo(1.2, 12); + expect(0.25 * distanceCompensatedSend(0.3, 0.25)).toBeCloseTo(0.3, 12); + expect(distanceCompensatedSend(0.3, 0.5)).toBeCloseTo(0.6, 12); + }); + + it('leaves the send alone at full distance gain', () => { + expect(distanceCompensatedSend(0.3, 1)).toBe(0.3); + expect(distanceCompensatedSend(0, 0.25)).toBe(0); + }); + + it('caps the boost, so the reverb falls with the direct sound beyond the cap', () => { + const atCap = 1 / MAX_SEND_DISTANCE_BOOST; + expect(distanceCompensatedSend(0.3, atCap)).toBeCloseTo(0.3 * MAX_SEND_DISTANCE_BOOST, 12); + expect(distanceCompensatedSend(0.3, atCap / 10)).toBeCloseTo(0.3 * MAX_SEND_DISTANCE_BOOST, 12); + // The floor of the default curve (10 km) would otherwise ask for 10 000 x. + const floor = profileDistanceGain(Number.MAX_VALUE, DEFAULT_SPATIAL_PROFILE); + expect(distanceCompensatedSend(1, floor)).toBe(MAX_SEND_DISTANCE_BOOST); + }); + + it('stays finite where a linear curve reaches silence', () => { + expect(distanceCompensatedSend(0.3, 0)).toBeCloseTo(0.3 * MAX_SEND_DISTANCE_BOOST, 12); + expect(distanceCompensatedSend(0.3, Number.NaN)).toBeCloseTo(0.3 * MAX_SEND_DISTANCE_BOOST, 12); + }); +}); diff --git a/src/audio/distanceModel.ts b/src/audio/distanceModel.ts index bad25bf4..a967da98 100644 --- a/src/audio/distanceModel.ts +++ b/src/audio/distanceModel.ts @@ -89,6 +89,29 @@ export function profileDistanceGain(distance: number, profile: SpatialProfile): return refDistance / (refDistance + rolloff * (clamped - refDistance)); } +/** + * The most a reverb send is raised to undo a source's distance gain (32 x is + * +30 dB). The engine takes a send after the source's gain and accepts any + * finite send gain, so it sets no limit of its own, and the curves give none + * that is usable: the default one floors at 1 / 10 000 and a linear one + * reaches 0, where the tap is silent and no boost can recover it. So this is a + * choice: under the default curve a room's reverberant level is even out to + * 32 m, and past that it falls with the direct sound. It is a cap on the + * boost, not on the send, so that distance is the same whatever the send. + */ +export const MAX_SEND_DISTANCE_BOOST = 32; + +/** + * The gain to give a send that is tapped after `distanceGain`, so that what + * reaches the reverb is `send` times the undistanced sound: send / distanceGain, + * with the boost capped at {@link MAX_SEND_DISTANCE_BOOST}. The direct sound + * still falls with distance, so the direct-to-reverb ratio does too. + */ +export function distanceCompensatedSend(send: number, distanceGain: number): number { + const boost = distanceGain > 0 ? Math.min(1 / distanceGain, MAX_SEND_DISTANCE_BOOST) : MAX_SEND_DISTANCE_BOOST; + return send * boost; +} + /** Euclidean distance between two 3D positions; missing/short vectors → 0 (co-located). */ export function distanceBetween( a?: readonly number[] | null, diff --git a/src/audio/effects/EffectChain.ts b/src/audio/effects/EffectChain.ts index 149b377f..fd881ff8 100644 --- a/src/audio/effects/EffectChain.ts +++ b/src/audio/effects/EffectChain.ts @@ -60,6 +60,8 @@ export class EffectChain { private downstream: Bus | null = null; /** Aux-send target fed in parallel with the primary output, if any. */ private sendTarget: Bus | null = null; + /** The gain of the feed into {@link sendTarget}. */ + private sendGain = 1; private constructor(cacophony: Cacophony, id: string | null, bus: Bus, mode: ChainMode) { this.cacophony = cacophony; @@ -127,6 +129,10 @@ export class EffectChain { if (this.bus.destroyed) { return; } + if (target && this.sendTarget === target && this.sendGain === gain) { + // Already fed at this level: the feed is refreshed on every listener step. + return; + } if (this.sendTarget && this.sendTarget !== target) { this.bus.disconnect(this.sendTarget); this.sendTarget = null; @@ -134,6 +140,7 @@ export class EffectChain { if (target) { this.bus.connect(target, gain); this.sendTarget = target; + this.sendGain = gain; } } diff --git a/src/audio/namedRoute.ts b/src/audio/namedRoute.ts index 0a0a00d9..4f31d8ce 100644 --- a/src/audio/namedRoute.ts +++ b/src/audio/namedRoute.ts @@ -7,9 +7,15 @@ import type { Bus } from 'cacophony'; export interface NamedRouteState { /** Named chain the source routes to (primary, or aux when {@link namedSend} is set). */ namedChain?: string; + /** The send the server asked for. */ namedSend?: number; /** Whether {@link namedChain} is currently applied as the source's own route. */ chainRouted?: boolean; + /** + * The gain the live aux send has in the engine: {@link namedSend}, raised to + * undo the source's distance gain when the send is tapped after it. + */ + sendGain?: number; } export interface NamedRoutable extends NamedRouteState { @@ -22,6 +28,7 @@ export function clearNamedRoute(source: NamedRoutable, master: Bus | undefined): const chain = source.namedChain; const routed = source.chainRouted; source.chainRouted = false; + source.sendGain = undefined; if (!routed || !chain) { return; } @@ -39,6 +46,9 @@ export function clearNamedRoute(source: NamedRoutable, master: Bus | undefined): /** * Route the source to `chain` (primary, or an aux send at `send`), first * undoing a different named route. A no-op when that route is already live. + * `compensatedSend` is the gain the engine gives the send, when that is not + * `send` itself (see {@link NamedRouteState.sendGain}); a live aux send to the same + * chain is re-gained in place, never removed and added again. * Returns the error when the chain cannot be routed to; the source then plays * dry and keeps the route it wants, so a later call can apply it. */ @@ -47,23 +57,33 @@ export function routeNamedChain( chain: string | undefined, send: number | undefined, master: Bus | undefined, + compensatedSend: number | undefined = send, ): unknown { - if (source.chainRouted && source.namedChain === chain && source.namedSend === send) { + // A primary route has no send gain, whatever the caller computed. + const sendGain = send === undefined ? undefined : compensatedSend; + const live = source.chainRouted && source.namedChain === chain; + if (live && source.namedSend === send && source.sendGain === sendGain) { return undefined; } - clearNamedRoute(source, master); + // The engine re-gains an existing send; only a different chain, or a change + // between a primary route and an aux send, needs the old route undone. + const regain = live && source.namedSend !== undefined && send !== undefined; + if (!regain) { + clearNamedRoute(source, master); + } source.namedChain = chain; source.namedSend = send; if (!chain) { return undefined; } try { - if (send !== undefined) { - source.routeTo(chain, send); + if (sendGain !== undefined) { + source.routeTo(chain, sendGain); } else { source.routeTo(chain); } source.chainRouted = true; + source.sendGain = sendGain; return undefined; } catch (error) { return error ?? new Error(`chain '${chain}' unavailable`); diff --git a/src/gmcp/Client/Media.test.ts b/src/gmcp/Client/Media.test.ts index 289da5cb..7fa9e1cd 100644 --- a/src/gmcp/Client/Media.test.ts +++ b/src/gmcp/Client/Media.test.ts @@ -24,6 +24,7 @@ vi.mock('../../audio/PositionalFoaRenderer', () => ({ }, })); +import { MAX_SEND_DISTANCE_BOOST } from '../../audio/distanceModel'; import { MEDIA_SEEK_TOLERANCE_MS, MediaService } from '../../audio/MediaService'; import { MediaPayloadError } from '../../audio/mediaPayloads'; import { VectorTweener } from '../../audio/vectorTween'; @@ -1996,6 +1997,205 @@ describe('GMCPClientMedia', () => { }); }); + describe('reverb send independent of distance', () => { + // Mongoose north 4 m: distance 4 from a listener at the origin, so the + // default inverse curve gives a distance gain of 0.25. + const fire = { + key: 'fire', + name: 'fire.ogg', + type: 'sound', + volume: 50, + is3d: true, + position: [0, 4, 0], + chain: 'room', + send: 0.3, + }; + + async function playFire(extra: Record = {}) { + const sound = createMockSound('https://media.example/fire.ogg'); + mockCreateSound.mockResolvedValue(sound); + await handler.handlePlay({ ...fire, ...extra } as GMCPMessageClientMediaPlay); + return sound; + } + + function lastSend(sound: MockSound): unknown[] { + return sound.routeTo.mock.calls[sound.routeTo.mock.calls.length - 1]; + } + + it('sends a positional sound at send / distance gain, leaving the direct level alone', async () => { + const sound = await playFire(); + + expect(sound.routeTo).toHaveBeenCalledOnce(); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(1.2, 9)]); + // volume x distance gain x effective send = volume x send + expect(sound.volume).toBeCloseTo(0.5 * 0.25, 9); + expect(sound.volume * (lastSend(sound)[1] as number)).toBeCloseTo(0.5 * 0.3, 9); + }); + + it('re-gains the send in place when the listener moves', async () => { + const sound = await playFire(); + + // 2 m from the source: distance gain 0.5. + client.media.setListenerPosition([0, 0, 2]); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.6, 9)]); + expect(sound.volume).toBeCloseTo(0.5 * 0.5, 9); + + // On top of it: no attenuation, so the send is the wire send. + client.media.setListenerPosition([0, 0, 4]); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.3, 9)]); + + expect(sound.removeSend).not.toHaveBeenCalled(); + expect(sound.voice.play).toHaveBeenCalledOnce(); + }); + + it('does not call the engine again when the listener moves without changing the distance', async () => { + const sound = await playFire(); + sound.routeTo.mockClear(); + + // Still 4 m away, on the other side. + client.media.setListenerPosition([0, 0, 8]); + expect(sound.routeTo).not.toHaveBeenCalled(); + }); + + it('follows the source as it glides', async () => { + const sound = await playFire(); + + handler.handleUpdate({ key: 'fire', position: [0, 2, 0] } as GMCPMessageClientMediaUpdate); + client.stepMotion(10_000); + + expect(sound.volume).toBeCloseTo(0.5 * 0.5, 9); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.6, 9)]); + }); + + it('caps the boost for a very distant sound', async () => { + const sound = await playFire(); + + client.media.setListenerPosition([0, 0, 1004]); + expect(sound.volume).toBeCloseTo(0.5 / 1000, 9); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.3 * MAX_SEND_DISTANCE_BOOST, 9)]); + }); + + it('stays finite where a linear profile reaches silence', async () => { + const sound = await playFire({ + spatial: { + model: 'linear', + refDistance: 1, + maxDistance: 4, + rolloff: 1, + coneInnerAngle: 360, + coneOuterAngle: 360, + coneOuterGain: 1, + }, + }); + + expect(sound.volume).toBe(0); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.3 * MAX_SEND_DISTANCE_BOOST, 9)]); + }); + + it('recomputes when an Update changes the send, without dropping the send', async () => { + const sound = await playFire(); + + handler.handleUpdate({ key: 'fire', send: 0.5 } as GMCPMessageClientMediaUpdate); + + expect(lastSend(sound)).toEqual(['room', expect.closeTo(2, 9)]); + expect(sound.removeSend).not.toHaveBeenCalled(); + }); + + it('recomputes when an Update changes the spatial profile', async () => { + const sound = await playFire(); + + handler.handleUpdate({ + key: 'fire', + spatial: { + model: 'inverse', + refDistance: 2, + maxDistance: 50, + rolloff: 1, + coneInnerAngle: 360, + coneOuterAngle: 360, + coneOuterGain: 1, + }, + } as GMCPMessageClientMediaUpdate); + + // 2 / (2 + (4 - 2)) = 0.5 + expect(sound.volume).toBeCloseTo(0.5 * 0.5, 9); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.6, 9)]); + }); + + it('carries the compensated send to a new chain', async () => { + const sound = await playFire(); + + handler.handleUpdate({ key: 'fire', chain: 'hall' } as GMCPMessageClientMediaUpdate); + + expect(sound.removeSend).toHaveBeenCalledWith('room'); + expect(lastSend(sound)).toEqual(['hall', expect.closeTo(1.2, 9)]); + }); + + it('leaves a non-positional sound at its wire send, wherever the listener goes', async () => { + const sound = await playFire({ is3d: false, position: undefined }); + + expect(sound.routeTo).toHaveBeenCalledOnce(); + expect(sound.routeTo).toHaveBeenCalledWith('room', 0.3); + + client.media.setListenerPosition([0, 0, 50]); + handler.handleUpdate({ key: 'fire', volume: 25 } as GMCPMessageClientMediaUpdate); + expect(sound.routeTo).toHaveBeenCalledOnce(); + expect(sound.volume).toBe(0.25); + }); + + it('makes no send call for a sound with no chain', async () => { + const sound = await playFire({ chain: undefined }); + + client.media.setListenerPosition([0, 0, 2]); + handler.handleUpdate({ key: 'fire', position: [0, 9, 0] } as GMCPMessageClientMediaUpdate); + client.stepMotion(10_000); + + expect(sound.routeTo).not.toHaveBeenCalled(); + expect(sound.removeSend).not.toHaveBeenCalled(); + }); + + it('leaves a primary (send-less) route alone', async () => { + const sound = await playFire({ send: undefined }); + + client.media.setListenerPosition([0, 0, 2]); + + expect(sound.routeTo).toHaveBeenCalledOnce(); + expect(sound.routeTo).toHaveBeenCalledWith('room'); + }); + + it('does not retry a send the engine refused each time the listener moves', async () => { + const consoleWarn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const sound = createMockSound('https://media.example/fire.ogg'); + sound.routeTo.mockImplementation(() => { + throw new Error("No bus registered with name 'room'"); + }); + mockCreateSound.mockResolvedValue(sound); + await handler.handlePlay(fire as GMCPMessageClientMediaPlay); + expect(sound.routeTo).toHaveBeenCalledOnce(); + + client.media.setListenerPosition([0, 0, 2]); + expect(sound.routeTo).toHaveBeenCalledOnce(); + consoleWarn.mockRestore(); + }); + + it('compensates the aux feed of an inline effect chain the same way', async () => { + await client.media.setChain({ id: 'room', effects: [{ type: 'reverb' }] }); + const room = client.effectBuses.created.room; + const sound = await playFire({ + effects: [{ id: 'muffle', type: 'lowpass', params: { frequency: 400 } }], + }); + const inline = client.effectBuses.anon[0]; + + expect(sound.routeTo).toHaveBeenCalledWith(inline); + expect(inline.connect).toHaveBeenLastCalledWith(room, expect.closeTo(1.2, 9)); + + client.media.setListenerPosition([0, 0, 2]); + expect(inline.connect).toHaveBeenLastCalledWith(room, expect.closeTo(0.6, 9)); + // The dry path stays on master throughout. + expect(inline.disconnect).not.toHaveBeenCalled(); + }); + }); + describe('per-voice occlusion', () => { const tone = { key: 's4894767', @@ -2767,12 +2967,15 @@ describe('GMCPClientMedia', () => { second.voice.play.mock.invocationCallOrder[0], ); expect(handler.sounds.fire.mediaVolume).toBe(0.2); - expect(second.routeTo).toHaveBeenLastCalledWith('workshop', 0.3); + expect(handler.sounds.fire.namedSend).toBe(0.3); expect(second.loop).toHaveBeenCalledWith('infinite'); if (mode === 'HRTF') { expect(second.position).toEqual([-2, 0, 0]); + // 2 m away: distance gain 0.5, which the send (tapped after it) undoes. + expect(second.routeTo).toHaveBeenLastCalledWith('workshop', expect.closeTo(0.6, 9)); } else { expect(second.volume).toBe(0.2); + expect(second.routeTo).toHaveBeenLastCalledWith('workshop', 0.3); } }); From c969b159a095228a8e847dac464fd8276e5d834c Mon Sep 17 00:00:00 2001 From: Christopher Toth Date: Sun, 4 Oct 2026 22:54:23 -0600 Subject: [PATCH 2/2] Give voices a facing, and keep their reverb send level with distance A voice in the scene was an omnidirectional point source: the speaking entity's forward vector arrives with Client.Spatial.EntityMove and is tweened into the spatial store, and nothing read it. A positional voice now takes its orientation from that vector, already in Web Audio axes, with a gentle cone: full level within 60 degrees either side of the speaker's facing, falling evenly to -6 dB directly behind (panner cone 120 / 360 / 0.501). The facing follows the entity through the store subscription that already moves the voice; a turn ramps like a move. An entity with no forward has no cone, and a non-positional voice has no panner at all, as before. The distance curve was already the one media sounds use by default: the voice panner took the same three constants DEFAULT_SPATIAL_PROFILE is built from. It is now derived from that profile by name (profilePannerDistance), so the two cannot drift apart. Web Audio's inverse model is the profile's formula exactly, so the rolloff stays in the panner and there is no separate distance gain for a voice. That rolloff is inside the panner, which is ahead of the gain node the engine taps a send from, so a voice's reverb send fell with distance just as a sound's did. The send is now given send / distanceGain with the same cap, where the distance gain is computed from the profile, the voice's position and the engine's listener position, since a panner's gain cannot be read back. It is re-gained in place when the speaker or the listener moves and when the send or chain changes. The cone is also ahead of the tap: a speaker facing away sends up to 6 dB less to the room. That is not compensated here. --- src/audio/LiveKitSpatialAudioBridge.test.ts | 83 +++++- src/audio/LiveKitSpatialAudioBridge.ts | 29 +- src/audio/MediaService.ts | 21 +- src/audio/MediaService.voices.test.ts | 15 + src/audio/MediaVoices.test.ts | 290 +++++++++++++++++++- src/audio/MediaVoices.ts | 164 +++++++++-- src/audio/distanceModel.test.ts | 26 ++ src/audio/distanceModel.ts | 21 ++ src/components/audioChat.tsx | 2 + 9 files changed, 613 insertions(+), 38 deletions(-) diff --git a/src/audio/LiveKitSpatialAudioBridge.test.ts b/src/audio/LiveKitSpatialAudioBridge.test.ts index 0cd792a4..b192a1d5 100644 --- a/src/audio/LiveKitSpatialAudioBridge.test.ts +++ b/src/audio/LiveKitSpatialAudioBridge.test.ts @@ -1,15 +1,17 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useSpatialStore } from "../stores/spatialStore"; import { LiveKitSpatialAudioBridge } from "./LiveKitSpatialAudioBridge"; function createMedia() { const voices: Array<{ detach: ReturnType; + setFacing: ReturnType; setPosition: ReturnType; setRoute: ReturnType; }> = []; const media = { attachVoice: vi.fn(() => { - const voice = { detach: vi.fn(), setPosition: vi.fn(), setRoute: vi.fn() }; + const voice = { detach: vi.fn(), setFacing: vi.fn(), setPosition: vi.fn(), setRoute: vi.fn() }; voices.push(voice); return voice; }), @@ -33,7 +35,8 @@ describe("LiveKitSpatialAudioBridge", () => { bridge.attachParticipantTrack("player-2", remoteTrack); - expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3]); + // No facing lookup was given: the voice has none. + expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3], null); }); it("attaches a participant who is not in the scene as a non-positional voice", () => { @@ -44,7 +47,7 @@ describe("LiveKitSpatialAudioBridge", () => { bridge.attachParticipantTrack("player-2", remoteTrack); // A phone-call partner has no place in this room: never the room origin. - expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, null); + expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, null, null); }); it("makes the voice positional when its entity appears, and non-positional when it leaves", () => { @@ -127,4 +130,78 @@ describe("LiveKitSpatialAudioBridge", () => { expect(voices[1].detach).toHaveBeenCalledOnce(); expect(voices[0].detach).toHaveBeenCalledOnce(); }); + + describe("the speaker's facing", () => { + it("attaches the voice facing the way its entity does", () => { + const { media } = createMedia(); + const remoteTrack = track("a"); + const bridge = new LiveKitSpatialAudioBridge( + media, + () => [1, 2, 3], + () => [0, 0, -1], + ); + + bridge.attachParticipantTrack("player-2", remoteTrack); + + expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3], [0, 0, -1]); + }); + + it("attaches an entity with no forward, and a bridge given no facing lookup, with no facing", () => { + const { media } = createMedia(); + const remoteTrack = track("a"); + + new LiveKitSpatialAudioBridge( + media, + () => [1, 2, 3], + () => undefined, + ).attachParticipantTrack("player-2", remoteTrack); + new LiveKitSpatialAudioBridge(media, () => [1, 2, 3]).attachParticipantTrack("player-3", remoteTrack); + + expect(media.attachVoice).toHaveBeenNthCalledWith(1, remoteTrack, [1, 2, 3], null); + expect(media.attachVoice).toHaveBeenNthCalledWith(2, remoteTrack, [1, 2, 3], null); + }); + + it("turns the voice on sync, and drops the facing when the forward goes", () => { + const { media, voices } = createMedia(); + const facings: Record = { "player-2": [0, 0, -1] }; + const bridge = new LiveKitSpatialAudioBridge( + media, + () => [1, 2, 3], + (participantId) => facings[participantId], + ); + bridge.attachParticipantTrack("player-2", track("a")); + + facings["player-2"] = [1, 0, 0]; + bridge.syncAll(); + expect(voices[0].setFacing).toHaveBeenLastCalledWith([1, 0, 0]); + + delete facings["player-2"]; + bridge.syncParticipant("player-2"); + expect(voices[0].setFacing).toHaveBeenLastCalledWith(null); + }); + + it("follows the entity through the spatial store, as the audio chat subscribes it", () => { + const store = useSpatialStore; + store.getState().reset(); + store.getState().enterEntity({ id: "player-2", position: [1, 2, 3], forward: [0, 0, -1] }); + const { media, voices } = createMedia(); + const bridge = new LiveKitSpatialAudioBridge( + media, + (participantId) => store.getState().spatialEntities[participantId]?.position, + (participantId) => store.getState().spatialEntities[participantId]?.forward, + ); + const unsubscribe = store.subscribe(() => bridge.syncAll()); + const remoteTrack = track("a"); + bridge.attachParticipantTrack("player-2", remoteTrack); + expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3], [0, 0, -1]); + + // A step of the turn Client.Spatial.EntityMove tweens into the store. + store.getState().patchEntity("player-2", { forward: [1, 0, 0] }); + expect(voices[0].setFacing).toHaveBeenLastCalledWith([1, 0, 0]); + expect(voices[0].setPosition).toHaveBeenLastCalledWith([1, 2, 3]); + + unsubscribe(); + store.getState().reset(); + }); + }); }); diff --git a/src/audio/LiveKitSpatialAudioBridge.ts b/src/audio/LiveKitSpatialAudioBridge.ts index 4b90dde1..790f7fc9 100644 --- a/src/audio/LiveKitSpatialAudioBridge.ts +++ b/src/audio/LiveKitSpatialAudioBridge.ts @@ -1,11 +1,14 @@ import type { Position } from "cacophony"; import type { MediaService } from "./MediaService"; -import type { MediaVoice, VoicePosition, VoiceRoute } from "./MediaVoices"; +import type { MediaVoice, VoiceFacing, VoicePosition, VoiceRoute } from "./MediaVoices"; /** The participant's position as an entity in the current scene; null or undefined when not in it. */ export type SpatialPositionLookup = (participantId: string) => Position | null | undefined; +/** The way the participant's entity faces (its `forward`); null or undefined when it has none. */ +export type SpatialFacingLookup = (participantId: string) => Position | null | undefined; + type VoiceMedia = Pick; interface SpatialAudioEntry { @@ -15,8 +18,8 @@ interface SpatialAudioEntry { /** * One LiveKit room's remote participants, as voices in the shared audio graph: - * positioned at their entity while it is in the current scene, non-positional - * otherwise. + * positioned at their entity, and facing the way it faces, while it is in the + * current scene; non-positional otherwise. */ export class LiveKitSpatialAudioBridge { private readonly entries = new Map(); @@ -25,6 +28,7 @@ export class LiveKitSpatialAudioBridge { constructor( private readonly media: VoiceMedia, private readonly lookupPosition: SpatialPositionLookup, + private readonly lookupFacing: SpatialFacingLookup = () => null, ) {} attachParticipantTrack(participantId: string, track: MediaStreamTrack): void { @@ -37,7 +41,11 @@ export class LiveKitSpatialAudioBridge { this.detachParticipant(participantId); // A first placement snaps, so a new voice does not fly in from the origin. - const voice = this.media.attachVoice(track, this.positionFor(participantId)); + const voice = this.media.attachVoice( + track, + this.positionFor(participantId), + this.facingFor(participantId), + ); voice.setRoute(this.route); this.entries.set(participantId, { track, voice }); } @@ -50,8 +58,14 @@ export class LiveKitSpatialAudioBridge { } } + /** Bring the participant's voice to where its entity is now, and the way it faces. */ syncParticipant(participantId: string): void { - this.entries.get(participantId)?.voice.setPosition(this.positionFor(participantId)); + const voice = this.entries.get(participantId)?.voice; + if (!voice) { + return; + } + voice.setPosition(this.positionFor(participantId)); + voice.setFacing(this.facingFor(participantId)); } syncAll(): void { @@ -93,4 +107,9 @@ export class LiveKitSpatialAudioBridge { private positionFor(participantId: string): VoicePosition { return this.lookupPosition(participantId) ?? null; } + + /** The way the participant's entity faces, or null: a voice with no facing has no cone. */ + private facingFor(participantId: string): VoiceFacing { + return this.lookupFacing(participantId) ?? null; + } } diff --git a/src/audio/MediaService.ts b/src/audio/MediaService.ts index 9bb91a5a..6d20e274 100644 --- a/src/audio/MediaService.ts +++ b/src/audio/MediaService.ts @@ -25,7 +25,12 @@ import { profileDistanceGain, type SpatialProfile, } from './distanceModel'; -import { type MediaVoice, MediaVoices, type VoicePosition } from './MediaVoices'; +import { + type MediaVoice, + MediaVoices, + type VoiceFacing, + type VoicePosition, +} from './MediaVoices'; import { clearNamedRoute, type NamedRouteState, routeNamedChain } from './namedRoute'; import { hasOcclusion, OCCLUSION_GLIDE_MS } from './occlusion'; import { VectorTweener } from './vectorTween'; @@ -352,6 +357,7 @@ export class MediaService { this.updatePositionalSpatial(sound); this.applyLevels(sound); } + this.voices.listenerMoved(); } } @@ -483,11 +489,16 @@ export class MediaService { /** * Put a live voice track (a LiveKit participant) in the graph: an HRTF point - * source at `position`, or non-positional when it is null. It is not a media - * key: Client.Media.Stop never touches it. + * source at `position` facing `forward` (no facing when that is null), or + * non-positional when the position is null. It is not a media key: + * Client.Media.Stop never touches it. */ - attachVoice(track: MediaStreamTrack, position: VoicePosition): MediaVoice { - return this.voices.attach(track, position); + attachVoice( + track: MediaStreamTrack, + position: VoicePosition, + forward: VoiceFacing = null, + ): MediaVoice { + return this.voices.attach(track, position, forward); } setChain(data: ClientMediaChainPayload): Promise { diff --git a/src/audio/MediaService.voices.test.ts b/src/audio/MediaService.voices.test.ts index b52b946a..602887f2 100644 --- a/src/audio/MediaService.voices.test.ts +++ b/src/audio/MediaService.voices.test.ts @@ -88,6 +88,21 @@ describe('MediaService voices', () => { expect(sound.routeTo).toHaveBeenLastCalledWith('room', 0.4); }); + it('keeps a voice send level with the room as the listener walks away from the speaker', async () => { + const { media, sound } = setup(); + await media.setChain(reverb); + media.setListenerPosition([0, 0, 0]); + const voice = media.attachVoice(track(), [0, 0, 4], [0, 0, -1]); + voice.setRoute({ chain: 'room', send: 0.3 }); + // 4 m: the panner's distance gain is 0.25, ahead of the tap. + expect(sound.routeTo).toHaveBeenLastCalledWith('room', expect.closeTo(1.2, 9)); + expect(sound.threeDOptions).toEqual(expect.objectContaining({ orientationZ: -1 })); + + media.setListenerPosition([0, 0, 2]); + expect(sound.routeTo).toHaveBeenLastCalledWith('room', expect.closeTo(0.6, 9)); + expect(sound.removeSend).not.toHaveBeenCalled(); + }); + it('keeps voices through a stop-all and a reset, which only returns them to dry', async () => { const { media, order, sound } = setup(); await media.setChain(reverb); diff --git a/src/audio/MediaVoices.test.ts b/src/audio/MediaVoices.test.ts index a5c08a9a..5be621d7 100644 --- a/src/audio/MediaVoices.test.ts +++ b/src/audio/MediaVoices.test.ts @@ -2,8 +2,17 @@ import type { Cacophony } from 'cacophony'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { SPATIAL_PARAM_TAU_S } from './audioParamSmoothing'; -import { SPATIAL_DISTANCE_MODEL } from './distanceModel'; -import { MediaVoices } from './MediaVoices'; +import { + DEFAULT_SPATIAL_PROFILE, + MAX_SEND_DISTANCE_BOOST, + SPATIAL_DISTANCE_MODEL, +} from './distanceModel'; +import { + MediaVoices, + VOICE_CONE_INNER_ANGLE, + VOICE_CONE_OUTER_ANGLE, + VOICE_CONE_OUTER_GAIN, +} from './MediaVoices'; const MockMediaStream = vi.fn(); vi.stubGlobal('MediaStream', MockMediaStream); @@ -53,6 +62,8 @@ function setup() { const made: ReturnType[] = []; const cacophony = { context: { currentTime: 7 }, + /** Unset unless a test places the listener: every voice is then at distance gain 1. */ + listenerPosition: undefined as [number, number, number] | undefined, createMediaStreamSound: vi.fn((_stream: unknown, options?: { panType?: 'HRTF' | 'stereo' }) => { const voice = makeVoiceSound(options?.panType); made.push(voice); @@ -329,4 +340,279 @@ describe('MediaVoices', () => { expect(sound.routeTo).toHaveBeenLastCalledWith('room', 0.2); expect(sound.removeSend).not.toHaveBeenCalled(); }); + + describe('distance', () => { + it('falls off on the curve media sounds use by default', () => { + const { made, voices } = setup(); + + voices.attach(track(), [1, 2, 3]); + + expect(made[0].sound.threeDOptions).toEqual( + expect.objectContaining({ + distanceModel: DEFAULT_SPATIAL_PROFILE.model, + refDistance: DEFAULT_SPATIAL_PROFILE.refDistance, + rolloffFactor: DEFAULT_SPATIAL_PROFILE.rolloff, + maxDistance: DEFAULT_SPATIAL_PROFILE.maxDistance, + }), + ); + }); + }); + + describe("a speaker's facing", () => { + const cone = { + coneInnerAngle: VOICE_CONE_INNER_ANGLE, + coneOuterAngle: VOICE_CONE_OUTER_ANGLE, + coneOuterGain: VOICE_CONE_OUTER_GAIN, + }; + const omnidirectional = { coneInnerAngle: 360, coneOuterAngle: 360, coneOuterGain: 1 }; + + it('is a gentle cone: full level within 60 degrees either side, -6 dB directly behind', () => { + expect(VOICE_CONE_INNER_ANGLE).toBe(120); + expect(VOICE_CONE_OUTER_ANGLE).toBe(360); + expect(20 * Math.log10(VOICE_CONE_OUTER_GAIN)).toBeCloseTo(-6, 9); + }); + + it('points the voice along its entity forward, with the cone, before it plays', () => { + const { cacophony, made, voices } = setup(); + let optionsAtPlay: unknown; + cacophony.createMediaStreamSound.mockImplementationOnce( + (_stream: unknown, options?: { panType?: 'HRTF' | 'stereo' }) => { + const voice = makeVoiceSound(options?.panType); + voice.sound.play.mockImplementation(() => { + optionsAtPlay = voice.sound.threeDOptions; + return []; + }); + made.push(voice); + return voice.sound; + }, + ); + + voices.attach(track(), [1, 2, 3], [0, 0, -1]); + + const expected = expect.objectContaining({ + ...cone, + orientationX: 0, + orientationY: 0, + orientationZ: -1, + // Still the positional voice it was: the cone is added, nothing replaced. + panningModel: 'HRTF', + refDistance: DEFAULT_SPATIAL_PROFILE.refDistance, + }); + expect(optionsAtPlay).toEqual(expected); + expect(made[0].sound.threeDOptions).toEqual(expected); + }); + + it('turns with the entity', () => { + const { made, voices } = setup(); + const voice = voices.attach(track(), [1, 2, 3], [0, 0, -1]); + const { sound } = made[0]; + + voice.setFacing([1, 0, 0]); + + expect(sound.threeDOptions).toEqual({ + ...cone, + orientationX: 1, + orientationY: 0, + orientationZ: 0, + }); + // The position is its own state: a turn does not move the voice. + expect(sound.position).toEqual([1, 2, 3]); + }); + + it('writes nothing to the engine when the facing has not changed', () => { + const { made, voices } = setup(); + const voice = voices.attach(track(), [1, 2, 3], [0, 0, -1]); + const { sound } = made[0]; + const before = sound.threeDOptions; + + voice.setFacing([0, 0, -1]); + + expect(sound.threeDOptions).toBe(before); + }); + + it('is omnidirectional for an entity with no forward', () => { + const { made, voices } = setup(); + + voices.attach(track(), [1, 2, 3]); + + const options = made[0].sound.threeDOptions as Record; + expect(options).toEqual(expect.objectContaining(omnidirectional)); + expect(options).not.toHaveProperty('orientationX'); + }); + + it('gains the cone when a forward arrives, and loses it when the forward goes', () => { + const { made, voices } = setup(); + const voice = voices.attach(track(), [1, 2, 3]); + const { sound } = made[0]; + + voice.setFacing([0, 0, 1]); + expect(sound.threeDOptions).toEqual({ + ...cone, + orientationX: 0, + orientationY: 0, + orientationZ: 1, + }); + + voice.setFacing(null); + expect(sound.threeDOptions).toEqual(omnidirectional); + }); + + it('treats a zero-length forward as no facing', () => { + const { made, voices } = setup(); + + voices.attach(track(), [1, 2, 3], [0, 0, 0]); + + expect(made[0].sound.threeDOptions).toEqual(expect.objectContaining(omnidirectional)); + }); + + it('gives a non-positional voice no cone, and keeps the facing for when it is placed', () => { + const { made, voices } = setup(); + const voice = voices.attach(track(), null, [0, 0, -1]); + + // A stereo voice has no panner: its setters would throw. + expect(made[0].sound.threeDOptions).toBeUndefined(); + voice.setFacing([1, 0, 0]); + expect(made[0].sound.threeDOptions).toBeUndefined(); + + voice.setPosition([1, 2, 3]); + expect(made[1].sound.threeDOptions).toEqual( + expect.objectContaining({ ...cone, orientationX: 1, orientationY: 0, orientationZ: 0 }), + ); + + voice.setPosition(null); + expect(made[2].sound.threeDOptions).toBeUndefined(); + }); + + it('ignores a turn after the voice is detached', () => { + const { made, voices } = setup(); + const voice = voices.attach(track(), [1, 2, 3]); + const before = made[0].sound.threeDOptions; + + voice.detach(); + voice.setFacing([1, 0, 0]); + + expect(made[0].sound.threeDOptions).toBe(before); + }); + }); + + describe('reverb send independent of distance', () => { + // The voice's rolloff is inside its panner, ahead of the gain the send is + // tapped from. 4 m from the listener: distance gain 0.25. + function room() { + const context = setup(); + context.defined.add('room'); + context.cacophony.listenerPosition = [0, 0, 0]; + return context; + } + + function lastSend(sound: ReturnType['sound']): unknown[] { + return sound.routeTo.mock.calls[sound.routeTo.mock.calls.length - 1]; + } + + it('sends a positional voice at send / distance gain', () => { + const { made, voices } = room(); + const voice = voices.attach(track(), [0, 0, 4]); + + voice.setRoute({ chain: 'room', send: 0.3 }); + + expect(lastSend(made[0].sound)).toEqual(['room', expect.closeTo(1.2, 9)]); + }); + + it('re-gains the send in place when the speaker moves', () => { + const { made, voices } = room(); + const voice = voices.attach(track(), [0, 0, 4]); + voice.setRoute({ chain: 'room', send: 0.3 }); + + voice.setPosition([0, 0, 2]); + + expect(lastSend(made[0].sound)).toEqual(['room', expect.closeTo(0.6, 9)]); + expect(made[0].sound.removeSend).not.toHaveBeenCalled(); + }); + + it('re-gains the send in place when the listener moves', () => { + const { cacophony, made, voices } = room(); + const voice = voices.attach(track(), [0, 0, 4]); + voice.setRoute({ chain: 'room', send: 0.3 }); + + cacophony.listenerPosition = [0, 0, 2]; + voices.listenerMoved(); + + expect(lastSend(made[0].sound)).toEqual(['room', expect.closeTo(0.6, 9)]); + expect(made[0].sound.removeSend).not.toHaveBeenCalled(); + }); + + it('caps the boost for a distant speaker', () => { + const { made, voices } = room(); + const voice = voices.attach(track(), [0, 0, 1000]); + + voice.setRoute({ chain: 'room', send: 0.3 }); + + expect(lastSend(made[0].sound)).toEqual([ + 'room', + expect.closeTo(0.3 * MAX_SEND_DISTANCE_BOOST, 9), + ]); + }); + + it('recomputes when the send changes', () => { + const { made, voices } = room(); + const voice = voices.attach(track(), [0, 0, 4]); + voice.setRoute({ chain: 'room', send: 0.3 }); + + voice.setRoute({ chain: 'room', send: 0.5 }); + + expect(lastSend(made[0].sound)).toEqual(['room', expect.closeTo(2, 9)]); + }); + + it('applies when the chain is defined after the voice asked for it', () => { + const { defined, made, voices } = room(); + defined.delete('room'); + const voice = voices.attach(track(), [0, 0, 4]); + voice.setRoute({ chain: 'room', send: 0.3 }); + expect(made[0].sound.routeTo).not.toHaveBeenCalled(); + + defined.add('room'); + voices.chainCreated('room'); + + expect(lastSend(made[0].sound)).toEqual(['room', expect.closeTo(1.2, 9)]); + }); + + it('leaves a non-positional voice at its send, wherever the listener goes', () => { + const { cacophony, made, voices } = room(); + const voice = voices.attach(track(), null); + voice.setRoute({ chain: 'room', send: 0.3 }); + + cacophony.listenerPosition = [0, 0, 50]; + voices.listenerMoved(); + + expect(made[0].sound.routeTo).toHaveBeenCalledOnce(); + expect(made[0].sound.routeTo).toHaveBeenCalledWith('room', 0.3); + }); + + it('follows the voice between positional and non-positional', () => { + const { made, voices } = room(); + const voice = voices.attach(track(), null); + voice.setRoute({ chain: 'room', send: 0.3 }); + + voice.setPosition([0, 0, 4]); + expect(lastSend(made[1].sound)).toEqual(['room', expect.closeTo(1.2, 9)]); + + voice.setPosition(null); + expect(lastSend(made[2].sound)).toEqual(['room', 0.3]); + }); + + it('makes no send call for a dry voice or a primary route', () => { + const { cacophony, made, voices } = room(); + voices.attach(track(), [0, 0, 4]); + const primary = voices.attach(track(), [0, 0, 4]); + primary.setRoute({ chain: 'room' }); + + cacophony.listenerPosition = [0, 0, 2]; + voices.listenerMoved(); + primary.setPosition([0, 0, 9]); + + expect(made[0].sound.routeTo).not.toHaveBeenCalled(); + expect(made[1].sound.routeTo).toHaveBeenCalledOnce(); + expect(made[1].sound.routeTo).toHaveBeenCalledWith('room'); + }); + }); }); diff --git a/src/audio/MediaVoices.ts b/src/audio/MediaVoices.ts index db88b31d..243eecdc 100644 --- a/src/audio/MediaVoices.ts +++ b/src/audio/MediaVoices.ts @@ -1,13 +1,20 @@ // Live voice streams (LiveKit participants) in the shared audio graph. A voice // whose speaker is an entity in the current scene is an HRTF point source at -// that entity; any other voice (a phone call, a speaker in another room) is -// non-positional. Either can play through a named effect chain, like a -// Client.Media sound, but lives as long as the call and is never a media key. +// that entity, facing the way the entity faces; any other voice (a phone call, +// a speaker in another room) is non-positional. Either can play through a +// named effect chain, like a Client.Media sound, but lives as long as the call +// and is never a media key. import type { Cacophony, Position } from 'cacophony'; import { SPATIAL_PARAM_TAU_S } from './audioParamSmoothing'; -import { SPATIAL_DISTANCE_MODEL } from './distanceModel'; +import { + DEFAULT_SPATIAL_PROFILE, + distanceBetween, + distanceCompensatedSend, + profileDistanceGain, + profilePannerDistance, +} from './distanceModel'; import { clearNamedRoute, type NamedRouteState, routeNamedChain } from './namedRoute'; /** The named chain a voice plays through: primary, or an aux send when `send` is set. */ @@ -19,6 +26,12 @@ export interface VoiceRoute { /** Where a voice is heard from; `null` is nowhere: non-positional. */ export type VoicePosition = Position | null; +/** + * The way a voice's speaker faces, in Web Audio axes; `null` is no facing, + * which is heard equally from every side. + */ +export type VoiceFacing = Position | null; + export interface MediaVoice { /** * Move the voice; the move is ramped over a short time constant. `null` @@ -26,6 +39,11 @@ export interface MediaVoice { * position after `null` places it there directly. */ setPosition(position: VoicePosition): void; + /** + * Turn the voice; the turn is ramped like a move. It is kept while the voice + * is non-positional, where it has no effect, for when the voice is placed. + */ + setFacing(forward: VoiceFacing): void; /** A full description: an absent chain means dry. */ setRoute(route: VoiceRoute): void; /** Remove the voice from the graph. The track is left running for its owner. */ @@ -38,23 +56,72 @@ export interface VoiceChains { hasPendingChain(id: string): boolean; } -type VoiceSound = ReturnType & NamedRouteState; +type VoiceSound = ReturnType & + NamedRouteState & { + /** Where the voice is; absent for a non-positional voice. */ + voicePosition?: Position; + }; + +/** + * A speaker's cone, as PannerNode cone parameters (angles are full widths, in + * degrees). Speech is directional but not sharply: the voice is at full level + * within 60° either side of where the speaker faces, and falls evenly from + * there to -6 dB directly behind. + */ +export const VOICE_CONE_INNER_ANGLE = 120; +/** The cone's falloff runs all the way round, so its lowest level is directly behind. */ +export const VOICE_CONE_OUTER_ANGLE = 360; +/** -6 dB as a linear gain. */ +export const VOICE_CONE_OUTER_GAIN = 10 ** (-6 / 20); + +/** + * Voices fall off with distance on the curve a media sound uses when it names + * none. The panner's own distance model expresses that curve exactly, so it is + * left to the panner; there is no separate distance gain for a voice. + */ +const VOICE_PROFILE = DEFAULT_SPATIAL_PROFILE; const VOICE_PANNER = { // A mono panner input: the browser downmixes a stereo track before panning. channelCount: 1, channelCountMode: 'explicit', channelInterpretation: 'speakers', - coneInnerAngle: 360, - coneOuterAngle: 360, - coneOuterGain: 0, - distanceModel: 'inverse', panningModel: 'HRTF', - refDistance: SPATIAL_DISTANCE_MODEL.refDistance, - rolloffFactor: SPATIAL_DISTANCE_MODEL.rolloffFactor, - maxDistance: SPATIAL_DISTANCE_MODEL.maxDistance, + ...profilePannerDistance(VOICE_PROFILE), } as const; +/** A usable facing: three finite components, not all zero. */ +function facingOf(forward: VoiceFacing | undefined): VoiceFacing { + if (!forward || forward.length < 3) { + return null; + } + const length = Math.hypot(forward[0], forward[1], forward[2]); + return Number.isFinite(length) && length > 0 ? [forward[0], forward[1], forward[2]] : null; +} + +function sameFacing(a: VoiceFacing, b: VoiceFacing): boolean { + return a === b || (a !== null && b !== null && a[0] === b[0] && a[1] === b[1] && a[2] === b[2]); +} + +/** The panner's cone and orientation for a facing: the voice cone, or none. */ +function facingOptions(facing: VoiceFacing) { + if (!facing) { + return { + coneInnerAngle: VOICE_PROFILE.coneInnerAngle, + coneOuterAngle: VOICE_PROFILE.coneOuterAngle, + coneOuterGain: VOICE_PROFILE.coneOuterGain, + }; + } + return { + coneInnerAngle: VOICE_CONE_INNER_ANGLE, + coneOuterAngle: VOICE_CONE_OUTER_ANGLE, + coneOuterGain: VOICE_CONE_OUTER_GAIN, + orientationX: facing[0], + orientationY: facing[1], + orientationZ: facing[2], + }; +} + export class MediaVoices { private readonly voices = new Set(); @@ -63,8 +130,9 @@ export class MediaVoices { private readonly chains: VoiceChains, ) {} - attach(track: MediaStreamTrack, position: VoicePosition): MediaVoice { - let sound = this.start(track, position); + attach(track: MediaStreamTrack, position: VoicePosition, forward: VoiceFacing = null): MediaVoice { + let facing = facingOf(forward); + let sound = this.start(track, position, facing); let positional = position !== null; void this.cacophony.resume().catch((error) => { @@ -77,10 +145,22 @@ export class MediaVoices { return; } if ((next !== null) !== positional) { - sound = this.restart(sound, track, next); + sound = this.restart(sound, track, next, facing); positional = next !== null; } else if (next !== null) { sound.position = next; + sound.voicePosition = next; + this.refreshSend(sound); + } + }, + setFacing: (next) => { + const turned = facingOf(next); + if (!this.voices.has(sound) || sameFacing(turned, facing)) { + return; + } + facing = turned; + if (positional) { + sound.threeDOptions = facingOptions(facing); } }, setRoute: (route) => { @@ -101,19 +181,21 @@ export class MediaVoices { * fixes a stream sound's panning mode when the sound is created. With a * position it is an HRTF point source. Without one it is a stereo source at * centre pan: the voice has no 3D panner at all, so it has no position, no - * distance model and no direction, wherever the listener stands or faces. + * distance model, no cone and no direction, wherever the listener stands or + * faces. */ - private start(track: MediaStreamTrack, position: VoicePosition): VoiceSound { + private start(track: MediaStreamTrack, position: VoicePosition, facing: VoiceFacing): VoiceSound { track.enabled = true; const sound: VoiceSound = this.cacophony.createMediaStreamSound(new MediaStream([track]), { panType: position ? 'HRTF' : 'stereo', stopTracksOnStop: false, }); if (position) { - sound.threeDOptions = VOICE_PANNER; + sound.threeDOptions = { ...VOICE_PANNER, ...facingOptions(facing) }; sound.position = position; - // The first pose is immediate; once playing, moves ramp, which de-zippers - // the per-frame steps the position tweener delivers. + sound.voicePosition = position; + // The first pose is immediate; once playing, moves and turns ramp, which + // de-zippers the per-frame steps the position and facing tweeners deliver. sound.spatialSmoothingTau = SPATIAL_PARAM_TAU_S; } sound.play(); @@ -127,11 +209,16 @@ export class MediaVoices { * track keeps running and the new sound takes over the route the old one * had, or still wanted. */ - private restart(old: VoiceSound, track: MediaStreamTrack, position: VoicePosition): VoiceSound { + private restart( + old: VoiceSound, + track: MediaStreamTrack, + position: VoicePosition, + facing: VoiceFacing, + ): VoiceSound { const { namedChain, namedSend } = old; this.voices.delete(old); old.cleanup(); - const sound = this.start(track, position); + const sound = this.start(track, position, facing); this.route(sound, namedChain, namedSend); return sound; } @@ -158,6 +245,13 @@ export class MediaVoices { } } + /** The listener moved: every positional voice's distance gain changed with it. */ + listenerMoved(): void { + for (const voice of this.voices) { + this.refreshSend(voice); + } + } + private route(voice: VoiceSound, chain: string | undefined, send: number | undefined): void { if (chain && !this.chains.hasChain(chain)) { // Not defined (yet): play dry and remember it for chainCreated. @@ -169,12 +263,36 @@ export class MediaVoices { } return; } - const error = routeNamedChain(voice, chain, send, this.master()); + const error = routeNamedChain(voice, chain, send, this.master(), this.sendGain(voice, send)); if (error) { console.warn(`Voice audio: chain '${chain}' unavailable; playing dry`, error); } } + /** + * The gain for a voice's send to its chain. A positional voice's distance + * rolloff is inside its panner, which is ahead of the gain node Cacophony + * taps a send from: left alone, a voice's reverb would fall with distance as + * fast as the voice itself. The panner's distance gain cannot be read back, + * so it is computed here from the same profile and the same positions the + * panner is given, and divided out of the send, as for a media sound. + * A non-positional voice has no distance gain. + */ + private sendGain(voice: VoiceSound, send: number | undefined): number | undefined { + if (send === undefined || !voice.voicePosition) { + return send; + } + const distance = distanceBetween(this.cacophony.listenerPosition, voice.voicePosition); + return distanceCompensatedSend(send, profileDistanceGain(distance, VOICE_PROFILE)); + } + + /** Bring a live send up to date with the voice's distance; a send that is not live is left alone. */ + private refreshSend(voice: VoiceSound): void { + if (voice.chainRouted && voice.namedSend !== undefined) { + this.route(voice, voice.namedChain, voice.namedSend); + } + } + private master() { return this.cacophony.getBus('master'); } diff --git a/src/audio/distanceModel.test.ts b/src/audio/distanceModel.test.ts index c84440f3..0365f18b 100644 --- a/src/audio/distanceModel.test.ts +++ b/src/audio/distanceModel.test.ts @@ -7,6 +7,7 @@ import { inverseDistanceGain, MAX_SEND_DISTANCE_BOOST, profileDistanceGain, + profilePannerDistance, SPATIAL_DISTANCE_MODEL, type SpatialProfile, } from './distanceModel'; @@ -103,6 +104,31 @@ describe('distanceBetween', () => { }); }); +describe('profilePannerDistance', () => { + it('gives the panner the default profile as its own inverse model', () => { + expect(profilePannerDistance(DEFAULT_SPATIAL_PROFILE)).toEqual({ + distanceModel: 'inverse', + refDistance: DEFAULT_SPATIAL_PROFILE.refDistance, + rolloffFactor: DEFAULT_SPATIAL_PROFILE.rolloff, + maxDistance: DEFAULT_SPATIAL_PROFILE.maxDistance, + }); + }); + + it('maps a linear profile to the linear model and none to no rolloff', () => { + const profile = { ...DEFAULT_SPATIAL_PROFILE, refDistance: 2, maxDistance: 40, rolloff: 0.5 }; + expect(profilePannerDistance({ ...profile, model: 'linear' })).toEqual({ + distanceModel: 'linear', + refDistance: 2, + rolloffFactor: 0.5, + maxDistance: 40, + }); + expect(profilePannerDistance({ ...profile, model: 'none' })).toMatchObject({ + distanceModel: 'inverse', + rolloffFactor: 0, + }); + }); +}); + describe('distanceCompensatedSend', () => { it('undoes the distance gain, so gain x send is the requested send', () => { expect(distanceCompensatedSend(0.3, 0.25)).toBeCloseTo(1.2, 12); diff --git a/src/audio/distanceModel.ts b/src/audio/distanceModel.ts index a967da98..383d4e49 100644 --- a/src/audio/distanceModel.ts +++ b/src/audio/distanceModel.ts @@ -89,6 +89,27 @@ export function profileDistanceGain(distance: number, profile: SpatialProfile): return refDistance / (refDistance + rolloff * (clamped - refDistance)); } +/** + * PannerNode distance parameters under which the panner's own rolloff is + * {@link profileDistanceGain} for the profile: Web Audio's `inverse` and + * `linear` models are the same formulas, with the same clamp at maxDistance. + * `none` is the inverse model with no rolloff. (Web Audio limits a linear + * rolloff to 0..1; a profile outside that range has no native equivalent.) + */ +export function profilePannerDistance(profile: SpatialProfile): { + distanceModel: 'inverse' | 'linear'; + refDistance: number; + rolloffFactor: number; + maxDistance: number; +} { + return { + distanceModel: profile.model === 'linear' ? 'linear' : 'inverse', + refDistance: profile.refDistance, + rolloffFactor: profile.model === 'none' ? 0 : profile.rolloff, + maxDistance: profile.maxDistance, + }; +} + /** * The most a reverb send is raised to undo a source's distance gain (32 x is * +30 dB). The engine takes a send after the source's gain and accepts any diff --git a/src/components/audioChat.tsx b/src/components/audioChat.tsx index 33b044d5..09755af2 100644 --- a/src/components/audioChat.tsx +++ b/src/components/audioChat.tsx @@ -45,6 +45,8 @@ const SpatialLiveKitAudio: React.FC = ({ client, ch bridgeRef.current = new LiveKitSpatialAudioBridge( client.media, (participantId) => useSpatialStore.getState().spatialEntities[participantId]?.position, + // Already in Web Audio axes: Client.Spatial converts and tweens it into the store. + (participantId) => useSpatialStore.getState().spatialEntities[participantId]?.forward, ); }