diff --git a/src/audio/LiveKitSpatialAudioBridge.test.ts b/src/audio/LiveKitSpatialAudioBridge.test.ts index 0cd792a4..b192a1d5 100644 --- a/src/audio/LiveKitSpatialAudioBridge.test.ts +++ b/src/audio/LiveKitSpatialAudioBridge.test.ts @@ -1,15 +1,17 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useSpatialStore } from "../stores/spatialStore"; import { LiveKitSpatialAudioBridge } from "./LiveKitSpatialAudioBridge"; function createMedia() { const voices: Array<{ detach: ReturnType; + setFacing: ReturnType; setPosition: ReturnType; setRoute: ReturnType; }> = []; const media = { attachVoice: vi.fn(() => { - const voice = { detach: vi.fn(), setPosition: vi.fn(), setRoute: vi.fn() }; + const voice = { detach: vi.fn(), setFacing: vi.fn(), setPosition: vi.fn(), setRoute: vi.fn() }; voices.push(voice); return voice; }), @@ -33,7 +35,8 @@ describe("LiveKitSpatialAudioBridge", () => { bridge.attachParticipantTrack("player-2", remoteTrack); - expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3]); + // No facing lookup was given: the voice has none. + expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3], null); }); it("attaches a participant who is not in the scene as a non-positional voice", () => { @@ -44,7 +47,7 @@ describe("LiveKitSpatialAudioBridge", () => { bridge.attachParticipantTrack("player-2", remoteTrack); // A phone-call partner has no place in this room: never the room origin. - expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, null); + expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, null, null); }); it("makes the voice positional when its entity appears, and non-positional when it leaves", () => { @@ -127,4 +130,78 @@ describe("LiveKitSpatialAudioBridge", () => { expect(voices[1].detach).toHaveBeenCalledOnce(); expect(voices[0].detach).toHaveBeenCalledOnce(); }); + + describe("the speaker's facing", () => { + it("attaches the voice facing the way its entity does", () => { + const { media } = createMedia(); + const remoteTrack = track("a"); + const bridge = new LiveKitSpatialAudioBridge( + media, + () => [1, 2, 3], + () => [0, 0, -1], + ); + + bridge.attachParticipantTrack("player-2", remoteTrack); + + expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3], [0, 0, -1]); + }); + + it("attaches an entity with no forward, and a bridge given no facing lookup, with no facing", () => { + const { media } = createMedia(); + const remoteTrack = track("a"); + + new LiveKitSpatialAudioBridge( + media, + () => [1, 2, 3], + () => undefined, + ).attachParticipantTrack("player-2", remoteTrack); + new LiveKitSpatialAudioBridge(media, () => [1, 2, 3]).attachParticipantTrack("player-3", remoteTrack); + + expect(media.attachVoice).toHaveBeenNthCalledWith(1, remoteTrack, [1, 2, 3], null); + expect(media.attachVoice).toHaveBeenNthCalledWith(2, remoteTrack, [1, 2, 3], null); + }); + + it("turns the voice on sync, and drops the facing when the forward goes", () => { + const { media, voices } = createMedia(); + const facings: Record = { "player-2": [0, 0, -1] }; + const bridge = new LiveKitSpatialAudioBridge( + media, + () => [1, 2, 3], + (participantId) => facings[participantId], + ); + bridge.attachParticipantTrack("player-2", track("a")); + + facings["player-2"] = [1, 0, 0]; + bridge.syncAll(); + expect(voices[0].setFacing).toHaveBeenLastCalledWith([1, 0, 0]); + + delete facings["player-2"]; + bridge.syncParticipant("player-2"); + expect(voices[0].setFacing).toHaveBeenLastCalledWith(null); + }); + + it("follows the entity through the spatial store, as the audio chat subscribes it", () => { + const store = useSpatialStore; + store.getState().reset(); + store.getState().enterEntity({ id: "player-2", position: [1, 2, 3], forward: [0, 0, -1] }); + const { media, voices } = createMedia(); + const bridge = new LiveKitSpatialAudioBridge( + media, + (participantId) => store.getState().spatialEntities[participantId]?.position, + (participantId) => store.getState().spatialEntities[participantId]?.forward, + ); + const unsubscribe = store.subscribe(() => bridge.syncAll()); + const remoteTrack = track("a"); + bridge.attachParticipantTrack("player-2", remoteTrack); + expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3], [0, 0, -1]); + + // A step of the turn Client.Spatial.EntityMove tweens into the store. + store.getState().patchEntity("player-2", { forward: [1, 0, 0] }); + expect(voices[0].setFacing).toHaveBeenLastCalledWith([1, 0, 0]); + expect(voices[0].setPosition).toHaveBeenLastCalledWith([1, 2, 3]); + + unsubscribe(); + store.getState().reset(); + }); + }); }); diff --git a/src/audio/LiveKitSpatialAudioBridge.ts b/src/audio/LiveKitSpatialAudioBridge.ts index 4b90dde1..790f7fc9 100644 --- a/src/audio/LiveKitSpatialAudioBridge.ts +++ b/src/audio/LiveKitSpatialAudioBridge.ts @@ -1,11 +1,14 @@ import type { Position } from "cacophony"; import type { MediaService } from "./MediaService"; -import type { MediaVoice, VoicePosition, VoiceRoute } from "./MediaVoices"; +import type { MediaVoice, VoiceFacing, VoicePosition, VoiceRoute } from "./MediaVoices"; /** The participant's position as an entity in the current scene; null or undefined when not in it. */ export type SpatialPositionLookup = (participantId: string) => Position | null | undefined; +/** The way the participant's entity faces (its `forward`); null or undefined when it has none. */ +export type SpatialFacingLookup = (participantId: string) => Position | null | undefined; + type VoiceMedia = Pick; interface SpatialAudioEntry { @@ -15,8 +18,8 @@ interface SpatialAudioEntry { /** * One LiveKit room's remote participants, as voices in the shared audio graph: - * positioned at their entity while it is in the current scene, non-positional - * otherwise. + * positioned at their entity, and facing the way it faces, while it is in the + * current scene; non-positional otherwise. */ export class LiveKitSpatialAudioBridge { private readonly entries = new Map(); @@ -25,6 +28,7 @@ export class LiveKitSpatialAudioBridge { constructor( private readonly media: VoiceMedia, private readonly lookupPosition: SpatialPositionLookup, + private readonly lookupFacing: SpatialFacingLookup = () => null, ) {} attachParticipantTrack(participantId: string, track: MediaStreamTrack): void { @@ -37,7 +41,11 @@ export class LiveKitSpatialAudioBridge { this.detachParticipant(participantId); // A first placement snaps, so a new voice does not fly in from the origin. - const voice = this.media.attachVoice(track, this.positionFor(participantId)); + const voice = this.media.attachVoice( + track, + this.positionFor(participantId), + this.facingFor(participantId), + ); voice.setRoute(this.route); this.entries.set(participantId, { track, voice }); } @@ -50,8 +58,14 @@ export class LiveKitSpatialAudioBridge { } } + /** Bring the participant's voice to where its entity is now, and the way it faces. */ syncParticipant(participantId: string): void { - this.entries.get(participantId)?.voice.setPosition(this.positionFor(participantId)); + const voice = this.entries.get(participantId)?.voice; + if (!voice) { + return; + } + voice.setPosition(this.positionFor(participantId)); + voice.setFacing(this.facingFor(participantId)); } syncAll(): void { @@ -93,4 +107,9 @@ export class LiveKitSpatialAudioBridge { private positionFor(participantId: string): VoicePosition { return this.lookupPosition(participantId) ?? null; } + + /** The way the participant's entity faces, or null: a voice with no facing has no cone. */ + private facingFor(participantId: string): VoiceFacing { + return this.lookupFacing(participantId) ?? null; + } } diff --git a/src/audio/MediaService.ts b/src/audio/MediaService.ts index b537f152..6d20e274 100644 --- a/src/audio/MediaService.ts +++ b/src/audio/MediaService.ts @@ -21,10 +21,16 @@ import type { PositionalFoaRenderer } from './PositionalFoaRenderer'; import { DEFAULT_SPATIAL_PROFILE, distanceBetween, + distanceCompensatedSend, profileDistanceGain, type SpatialProfile, } from './distanceModel'; -import { type MediaVoice, MediaVoices, type VoicePosition } from './MediaVoices'; +import { + type MediaVoice, + MediaVoices, + type VoiceFacing, + type VoicePosition, +} from './MediaVoices'; import { clearNamedRoute, type NamedRouteState, routeNamedChain } from './namedRoute'; import { hasOcclusion, OCCLUSION_GLIDE_MS } from './occlusion'; import { VectorTweener } from './vectorTween'; @@ -351,6 +357,7 @@ export class MediaService { this.updatePositionalSpatial(sound); this.applyLevels(sound); } + this.voices.listenerMoved(); } } @@ -385,11 +392,24 @@ export class MediaService { sound.ambisonicRenderer.setDistanceGain(this.distanceGain(sound)); } + /** + * The distance gain that is part of the sound's own gain: its profile's, for + * an HRTF point source, and 1 for every other sound (non-positional sounds + * have none; ambisonic routes apply distance in their renderer instead). + */ + private levelDistanceGain(sound: ExtendedSound): number { + return sound.pointSource && sound.upmix !== 'ambisonic' ? this.distanceGain(sound) : 1; + } + /** * Set the sound's gain: wire volume × 10^(gainDb/20) × (for an HRTF point * source) its profile distance gain. This is the one stage where distance * falloff is applied on the HRTF route; the panner's rolloff is 0. Ambisonic * routes apply distance in their renderer instead. + * + * The engine taps a send after this gain, so the send to the sound's chain + * is re-gained here too: every change of the distance gain (the listener or + * the source moves, the profile changes) comes through this method. */ private applyLevels(sound: ExtendedSound): void { if (this.cleanedSounds.has(sound)) { @@ -397,8 +417,40 @@ export class MediaService { } const volume = sound.mediaVolume ?? 1; const gain = sound.gainDb ? 10 ** (sound.gainDb / 20) : 1; - const distance = sound.pointSource && sound.upmix !== 'ambisonic' ? this.distanceGain(sound) : 1; - sound.volume = volume * gain * distance; + sound.volume = volume * gain * this.levelDistanceGain(sound); + this.refreshSend(sound); + } + + /** + * The gain for the sound's send to its named chain. Cacophony takes a send + * from the voice's output, after its gain, so the wire `send` alone would + * make the reverberant level fall with distance exactly as the direct level + * does. A room's reverberant field is roughly even, and the direct-to-reverb + * ratio is how a listener hears distance: so the distance gain is divided + * back out of the send ({@link distanceCompensatedSend}) and the chain gets + * volume × send wherever the source is. Occlusion sits before the tap and + * still dims direct and reverb alike. + */ + private sendGain(sound: ExtendedSound, send: number | undefined): number | undefined { + return send === undefined + ? undefined + : distanceCompensatedSend(send, this.levelDistanceGain(sound)); + } + + /** + * Bring a live send up to date with the sound's distance gain. Only a send + * that is already routed is touched: one the engine refused is not retried + * on every listener step. + */ + private refreshSend(sound: ExtendedSound): void { + if (sound.namedSend === undefined) { + return; + } + if (sound.effectChain) { + this.pointInlineChain(sound.effectChain, sound.namedChain, this.sendGain(sound, sound.namedSend)); + } else if (sound.chainRouted) { + this.routeNamedChain(sound, sound.namedChain, sound.namedSend); + } } /** HRTF panner settings for a point source: position and cone only, no native rolloff. */ @@ -437,11 +489,16 @@ export class MediaService { /** * Put a live voice track (a LiveKit participant) in the graph: an HRTF point - * source at `position`, or non-positional when it is null. It is not a media - * key: Client.Media.Stop never touches it. + * source at `position` facing `forward` (no facing when that is null), or + * non-positional when the position is null. It is not a media key: + * Client.Media.Stop never touches it. */ - attachVoice(track: MediaStreamTrack, position: VoicePosition): MediaVoice { - return this.voices.attach(track, position); + attachVoice( + track: MediaStreamTrack, + position: VoicePosition, + forward: VoiceFacing = null, + ): MediaVoice { + return this.voices.attach(track, position, forward); } setChain(data: ClientMediaChainPayload): Promise { @@ -1130,7 +1187,13 @@ export class MediaService { * undoing a different named route. A no-op when that route is already live. */ private routeNamedChain(sound: ExtendedSound, chain: string | undefined, send: number | undefined): void { - const error = routeNamedChain(sound, chain, send, this.cacophony.getBus('master')); + const error = routeNamedChain( + sound, + chain, + send, + this.cacophony.getBus('master'), + this.sendGain(sound, send), + ); if (error) { console.warn(`Client.Media: chain '${chain}' unavailable; playing dry`, error); this.traceSound('routed', sound, sound.key, { @@ -1195,7 +1258,7 @@ export class MediaService { // Update without effects: keep the inline chain, re-point what it feeds. sound.namedChain = chain; sound.namedSend = send; - this.pointInlineChain(sound.effectChain, chain, send); + this.pointInlineChain(sound.effectChain, chain, this.sendGain(sound, send)); return; } this.routeNamedChain(sound, chain, send); @@ -1237,6 +1300,8 @@ export class MediaService { * inline output stays on master (the dry path) and feeds the named chain at * the send level, exactly as chain+send behaves without inline effects. * Without a send, the inline chain runs in series into the named chain. + * `send` is the gain for that feed ({@link sendGain}): the inline bus is + * downstream of the sound's gain, so its feed is after the distance gain too. */ private pointInlineChain(inline: EffectChain, chain: string | undefined, send: number | undefined): void { const target = chain ? (this.effects.getChain(chain)?.bus ?? null) : null; @@ -1258,7 +1323,7 @@ export class MediaService { if (!inline) { return; } - this.pointInlineChain(inline, data.chain, data.send); + this.pointInlineChain(inline, data.chain, this.sendGain(sound, data.send)); try { sound.routeTo(inline.bus); } catch (error) { diff --git a/src/audio/MediaService.voices.test.ts b/src/audio/MediaService.voices.test.ts index b52b946a..602887f2 100644 --- a/src/audio/MediaService.voices.test.ts +++ b/src/audio/MediaService.voices.test.ts @@ -88,6 +88,21 @@ describe('MediaService voices', () => { expect(sound.routeTo).toHaveBeenLastCalledWith('room', 0.4); }); + it('keeps a voice send level with the room as the listener walks away from the speaker', async () => { + const { media, sound } = setup(); + await media.setChain(reverb); + media.setListenerPosition([0, 0, 0]); + const voice = media.attachVoice(track(), [0, 0, 4], [0, 0, -1]); + voice.setRoute({ chain: 'room', send: 0.3 }); + // 4 m: the panner's distance gain is 0.25, ahead of the tap. + expect(sound.routeTo).toHaveBeenLastCalledWith('room', expect.closeTo(1.2, 9)); + expect(sound.threeDOptions).toEqual(expect.objectContaining({ orientationZ: -1 })); + + media.setListenerPosition([0, 0, 2]); + expect(sound.routeTo).toHaveBeenLastCalledWith('room', expect.closeTo(0.6, 9)); + expect(sound.removeSend).not.toHaveBeenCalled(); + }); + it('keeps voices through a stop-all and a reset, which only returns them to dry', async () => { const { media, order, sound } = setup(); await media.setChain(reverb); diff --git a/src/audio/MediaVoices.test.ts b/src/audio/MediaVoices.test.ts index a5c08a9a..5be621d7 100644 --- a/src/audio/MediaVoices.test.ts +++ b/src/audio/MediaVoices.test.ts @@ -2,8 +2,17 @@ import type { Cacophony } from 'cacophony'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { SPATIAL_PARAM_TAU_S } from './audioParamSmoothing'; -import { SPATIAL_DISTANCE_MODEL } from './distanceModel'; -import { MediaVoices } from './MediaVoices'; +import { + DEFAULT_SPATIAL_PROFILE, + MAX_SEND_DISTANCE_BOOST, + SPATIAL_DISTANCE_MODEL, +} from './distanceModel'; +import { + MediaVoices, + VOICE_CONE_INNER_ANGLE, + VOICE_CONE_OUTER_ANGLE, + VOICE_CONE_OUTER_GAIN, +} from './MediaVoices'; const MockMediaStream = vi.fn(); vi.stubGlobal('MediaStream', MockMediaStream); @@ -53,6 +62,8 @@ function setup() { const made: ReturnType[] = []; const cacophony = { context: { currentTime: 7 }, + /** Unset unless a test places the listener: every voice is then at distance gain 1. */ + listenerPosition: undefined as [number, number, number] | undefined, createMediaStreamSound: vi.fn((_stream: unknown, options?: { panType?: 'HRTF' | 'stereo' }) => { const voice = makeVoiceSound(options?.panType); made.push(voice); @@ -329,4 +340,279 @@ describe('MediaVoices', () => { expect(sound.routeTo).toHaveBeenLastCalledWith('room', 0.2); expect(sound.removeSend).not.toHaveBeenCalled(); }); + + describe('distance', () => { + it('falls off on the curve media sounds use by default', () => { + const { made, voices } = setup(); + + voices.attach(track(), [1, 2, 3]); + + expect(made[0].sound.threeDOptions).toEqual( + expect.objectContaining({ + distanceModel: DEFAULT_SPATIAL_PROFILE.model, + refDistance: DEFAULT_SPATIAL_PROFILE.refDistance, + rolloffFactor: DEFAULT_SPATIAL_PROFILE.rolloff, + maxDistance: DEFAULT_SPATIAL_PROFILE.maxDistance, + }), + ); + }); + }); + + describe("a speaker's facing", () => { + const cone = { + coneInnerAngle: VOICE_CONE_INNER_ANGLE, + coneOuterAngle: VOICE_CONE_OUTER_ANGLE, + coneOuterGain: VOICE_CONE_OUTER_GAIN, + }; + const omnidirectional = { coneInnerAngle: 360, coneOuterAngle: 360, coneOuterGain: 1 }; + + it('is a gentle cone: full level within 60 degrees either side, -6 dB directly behind', () => { + expect(VOICE_CONE_INNER_ANGLE).toBe(120); + expect(VOICE_CONE_OUTER_ANGLE).toBe(360); + expect(20 * Math.log10(VOICE_CONE_OUTER_GAIN)).toBeCloseTo(-6, 9); + }); + + it('points the voice along its entity forward, with the cone, before it plays', () => { + const { cacophony, made, voices } = setup(); + let optionsAtPlay: unknown; + cacophony.createMediaStreamSound.mockImplementationOnce( + (_stream: unknown, options?: { panType?: 'HRTF' | 'stereo' }) => { + const voice = makeVoiceSound(options?.panType); + voice.sound.play.mockImplementation(() => { + optionsAtPlay = voice.sound.threeDOptions; + return []; + }); + made.push(voice); + return voice.sound; + }, + ); + + voices.attach(track(), [1, 2, 3], [0, 0, -1]); + + const expected = expect.objectContaining({ + ...cone, + orientationX: 0, + orientationY: 0, + orientationZ: -1, + // Still the positional voice it was: the cone is added, nothing replaced. + panningModel: 'HRTF', + refDistance: DEFAULT_SPATIAL_PROFILE.refDistance, + }); + expect(optionsAtPlay).toEqual(expected); + expect(made[0].sound.threeDOptions).toEqual(expected); + }); + + it('turns with the entity', () => { + const { made, voices } = setup(); + const voice = voices.attach(track(), [1, 2, 3], [0, 0, -1]); + const { sound } = made[0]; + + voice.setFacing([1, 0, 0]); + + expect(sound.threeDOptions).toEqual({ + ...cone, + orientationX: 1, + orientationY: 0, + orientationZ: 0, + }); + // The position is its own state: a turn does not move the voice. + expect(sound.position).toEqual([1, 2, 3]); + }); + + it('writes nothing to the engine when the facing has not changed', () => { + const { made, voices } = setup(); + const voice = voices.attach(track(), [1, 2, 3], [0, 0, -1]); + const { sound } = made[0]; + const before = sound.threeDOptions; + + voice.setFacing([0, 0, -1]); + + expect(sound.threeDOptions).toBe(before); + }); + + it('is omnidirectional for an entity with no forward', () => { + const { made, voices } = setup(); + + voices.attach(track(), [1, 2, 3]); + + const options = made[0].sound.threeDOptions as Record; + expect(options).toEqual(expect.objectContaining(omnidirectional)); + expect(options).not.toHaveProperty('orientationX'); + }); + + it('gains the cone when a forward arrives, and loses it when the forward goes', () => { + const { made, voices } = setup(); + const voice = voices.attach(track(), [1, 2, 3]); + const { sound } = made[0]; + + voice.setFacing([0, 0, 1]); + expect(sound.threeDOptions).toEqual({ + ...cone, + orientationX: 0, + orientationY: 0, + orientationZ: 1, + }); + + voice.setFacing(null); + expect(sound.threeDOptions).toEqual(omnidirectional); + }); + + it('treats a zero-length forward as no facing', () => { + const { made, voices } = setup(); + + voices.attach(track(), [1, 2, 3], [0, 0, 0]); + + expect(made[0].sound.threeDOptions).toEqual(expect.objectContaining(omnidirectional)); + }); + + it('gives a non-positional voice no cone, and keeps the facing for when it is placed', () => { + const { made, voices } = setup(); + const voice = voices.attach(track(), null, [0, 0, -1]); + + // A stereo voice has no panner: its setters would throw. + expect(made[0].sound.threeDOptions).toBeUndefined(); + voice.setFacing([1, 0, 0]); + expect(made[0].sound.threeDOptions).toBeUndefined(); + + voice.setPosition([1, 2, 3]); + expect(made[1].sound.threeDOptions).toEqual( + expect.objectContaining({ ...cone, orientationX: 1, orientationY: 0, orientationZ: 0 }), + ); + + voice.setPosition(null); + expect(made[2].sound.threeDOptions).toBeUndefined(); + }); + + it('ignores a turn after the voice is detached', () => { + const { made, voices } = setup(); + const voice = voices.attach(track(), [1, 2, 3]); + const before = made[0].sound.threeDOptions; + + voice.detach(); + voice.setFacing([1, 0, 0]); + + expect(made[0].sound.threeDOptions).toBe(before); + }); + }); + + describe('reverb send independent of distance', () => { + // The voice's rolloff is inside its panner, ahead of the gain the send is + // tapped from. 4 m from the listener: distance gain 0.25. + function room() { + const context = setup(); + context.defined.add('room'); + context.cacophony.listenerPosition = [0, 0, 0]; + return context; + } + + function lastSend(sound: ReturnType['sound']): unknown[] { + return sound.routeTo.mock.calls[sound.routeTo.mock.calls.length - 1]; + } + + it('sends a positional voice at send / distance gain', () => { + const { made, voices } = room(); + const voice = voices.attach(track(), [0, 0, 4]); + + voice.setRoute({ chain: 'room', send: 0.3 }); + + expect(lastSend(made[0].sound)).toEqual(['room', expect.closeTo(1.2, 9)]); + }); + + it('re-gains the send in place when the speaker moves', () => { + const { made, voices } = room(); + const voice = voices.attach(track(), [0, 0, 4]); + voice.setRoute({ chain: 'room', send: 0.3 }); + + voice.setPosition([0, 0, 2]); + + expect(lastSend(made[0].sound)).toEqual(['room', expect.closeTo(0.6, 9)]); + expect(made[0].sound.removeSend).not.toHaveBeenCalled(); + }); + + it('re-gains the send in place when the listener moves', () => { + const { cacophony, made, voices } = room(); + const voice = voices.attach(track(), [0, 0, 4]); + voice.setRoute({ chain: 'room', send: 0.3 }); + + cacophony.listenerPosition = [0, 0, 2]; + voices.listenerMoved(); + + expect(lastSend(made[0].sound)).toEqual(['room', expect.closeTo(0.6, 9)]); + expect(made[0].sound.removeSend).not.toHaveBeenCalled(); + }); + + it('caps the boost for a distant speaker', () => { + const { made, voices } = room(); + const voice = voices.attach(track(), [0, 0, 1000]); + + voice.setRoute({ chain: 'room', send: 0.3 }); + + expect(lastSend(made[0].sound)).toEqual([ + 'room', + expect.closeTo(0.3 * MAX_SEND_DISTANCE_BOOST, 9), + ]); + }); + + it('recomputes when the send changes', () => { + const { made, voices } = room(); + const voice = voices.attach(track(), [0, 0, 4]); + voice.setRoute({ chain: 'room', send: 0.3 }); + + voice.setRoute({ chain: 'room', send: 0.5 }); + + expect(lastSend(made[0].sound)).toEqual(['room', expect.closeTo(2, 9)]); + }); + + it('applies when the chain is defined after the voice asked for it', () => { + const { defined, made, voices } = room(); + defined.delete('room'); + const voice = voices.attach(track(), [0, 0, 4]); + voice.setRoute({ chain: 'room', send: 0.3 }); + expect(made[0].sound.routeTo).not.toHaveBeenCalled(); + + defined.add('room'); + voices.chainCreated('room'); + + expect(lastSend(made[0].sound)).toEqual(['room', expect.closeTo(1.2, 9)]); + }); + + it('leaves a non-positional voice at its send, wherever the listener goes', () => { + const { cacophony, made, voices } = room(); + const voice = voices.attach(track(), null); + voice.setRoute({ chain: 'room', send: 0.3 }); + + cacophony.listenerPosition = [0, 0, 50]; + voices.listenerMoved(); + + expect(made[0].sound.routeTo).toHaveBeenCalledOnce(); + expect(made[0].sound.routeTo).toHaveBeenCalledWith('room', 0.3); + }); + + it('follows the voice between positional and non-positional', () => { + const { made, voices } = room(); + const voice = voices.attach(track(), null); + voice.setRoute({ chain: 'room', send: 0.3 }); + + voice.setPosition([0, 0, 4]); + expect(lastSend(made[1].sound)).toEqual(['room', expect.closeTo(1.2, 9)]); + + voice.setPosition(null); + expect(lastSend(made[2].sound)).toEqual(['room', 0.3]); + }); + + it('makes no send call for a dry voice or a primary route', () => { + const { cacophony, made, voices } = room(); + voices.attach(track(), [0, 0, 4]); + const primary = voices.attach(track(), [0, 0, 4]); + primary.setRoute({ chain: 'room' }); + + cacophony.listenerPosition = [0, 0, 2]; + voices.listenerMoved(); + primary.setPosition([0, 0, 9]); + + expect(made[0].sound.routeTo).not.toHaveBeenCalled(); + expect(made[1].sound.routeTo).toHaveBeenCalledOnce(); + expect(made[1].sound.routeTo).toHaveBeenCalledWith('room'); + }); + }); }); diff --git a/src/audio/MediaVoices.ts b/src/audio/MediaVoices.ts index db88b31d..243eecdc 100644 --- a/src/audio/MediaVoices.ts +++ b/src/audio/MediaVoices.ts @@ -1,13 +1,20 @@ // Live voice streams (LiveKit participants) in the shared audio graph. A voice // whose speaker is an entity in the current scene is an HRTF point source at -// that entity; any other voice (a phone call, a speaker in another room) is -// non-positional. Either can play through a named effect chain, like a -// Client.Media sound, but lives as long as the call and is never a media key. +// that entity, facing the way the entity faces; any other voice (a phone call, +// a speaker in another room) is non-positional. Either can play through a +// named effect chain, like a Client.Media sound, but lives as long as the call +// and is never a media key. import type { Cacophony, Position } from 'cacophony'; import { SPATIAL_PARAM_TAU_S } from './audioParamSmoothing'; -import { SPATIAL_DISTANCE_MODEL } from './distanceModel'; +import { + DEFAULT_SPATIAL_PROFILE, + distanceBetween, + distanceCompensatedSend, + profileDistanceGain, + profilePannerDistance, +} from './distanceModel'; import { clearNamedRoute, type NamedRouteState, routeNamedChain } from './namedRoute'; /** The named chain a voice plays through: primary, or an aux send when `send` is set. */ @@ -19,6 +26,12 @@ export interface VoiceRoute { /** Where a voice is heard from; `null` is nowhere: non-positional. */ export type VoicePosition = Position | null; +/** + * The way a voice's speaker faces, in Web Audio axes; `null` is no facing, + * which is heard equally from every side. + */ +export type VoiceFacing = Position | null; + export interface MediaVoice { /** * Move the voice; the move is ramped over a short time constant. `null` @@ -26,6 +39,11 @@ export interface MediaVoice { * position after `null` places it there directly. */ setPosition(position: VoicePosition): void; + /** + * Turn the voice; the turn is ramped like a move. It is kept while the voice + * is non-positional, where it has no effect, for when the voice is placed. + */ + setFacing(forward: VoiceFacing): void; /** A full description: an absent chain means dry. */ setRoute(route: VoiceRoute): void; /** Remove the voice from the graph. The track is left running for its owner. */ @@ -38,23 +56,72 @@ export interface VoiceChains { hasPendingChain(id: string): boolean; } -type VoiceSound = ReturnType & NamedRouteState; +type VoiceSound = ReturnType & + NamedRouteState & { + /** Where the voice is; absent for a non-positional voice. */ + voicePosition?: Position; + }; + +/** + * A speaker's cone, as PannerNode cone parameters (angles are full widths, in + * degrees). Speech is directional but not sharply: the voice is at full level + * within 60° either side of where the speaker faces, and falls evenly from + * there to -6 dB directly behind. + */ +export const VOICE_CONE_INNER_ANGLE = 120; +/** The cone's falloff runs all the way round, so its lowest level is directly behind. */ +export const VOICE_CONE_OUTER_ANGLE = 360; +/** -6 dB as a linear gain. */ +export const VOICE_CONE_OUTER_GAIN = 10 ** (-6 / 20); + +/** + * Voices fall off with distance on the curve a media sound uses when it names + * none. The panner's own distance model expresses that curve exactly, so it is + * left to the panner; there is no separate distance gain for a voice. + */ +const VOICE_PROFILE = DEFAULT_SPATIAL_PROFILE; const VOICE_PANNER = { // A mono panner input: the browser downmixes a stereo track before panning. channelCount: 1, channelCountMode: 'explicit', channelInterpretation: 'speakers', - coneInnerAngle: 360, - coneOuterAngle: 360, - coneOuterGain: 0, - distanceModel: 'inverse', panningModel: 'HRTF', - refDistance: SPATIAL_DISTANCE_MODEL.refDistance, - rolloffFactor: SPATIAL_DISTANCE_MODEL.rolloffFactor, - maxDistance: SPATIAL_DISTANCE_MODEL.maxDistance, + ...profilePannerDistance(VOICE_PROFILE), } as const; +/** A usable facing: three finite components, not all zero. */ +function facingOf(forward: VoiceFacing | undefined): VoiceFacing { + if (!forward || forward.length < 3) { + return null; + } + const length = Math.hypot(forward[0], forward[1], forward[2]); + return Number.isFinite(length) && length > 0 ? [forward[0], forward[1], forward[2]] : null; +} + +function sameFacing(a: VoiceFacing, b: VoiceFacing): boolean { + return a === b || (a !== null && b !== null && a[0] === b[0] && a[1] === b[1] && a[2] === b[2]); +} + +/** The panner's cone and orientation for a facing: the voice cone, or none. */ +function facingOptions(facing: VoiceFacing) { + if (!facing) { + return { + coneInnerAngle: VOICE_PROFILE.coneInnerAngle, + coneOuterAngle: VOICE_PROFILE.coneOuterAngle, + coneOuterGain: VOICE_PROFILE.coneOuterGain, + }; + } + return { + coneInnerAngle: VOICE_CONE_INNER_ANGLE, + coneOuterAngle: VOICE_CONE_OUTER_ANGLE, + coneOuterGain: VOICE_CONE_OUTER_GAIN, + orientationX: facing[0], + orientationY: facing[1], + orientationZ: facing[2], + }; +} + export class MediaVoices { private readonly voices = new Set(); @@ -63,8 +130,9 @@ export class MediaVoices { private readonly chains: VoiceChains, ) {} - attach(track: MediaStreamTrack, position: VoicePosition): MediaVoice { - let sound = this.start(track, position); + attach(track: MediaStreamTrack, position: VoicePosition, forward: VoiceFacing = null): MediaVoice { + let facing = facingOf(forward); + let sound = this.start(track, position, facing); let positional = position !== null; void this.cacophony.resume().catch((error) => { @@ -77,10 +145,22 @@ export class MediaVoices { return; } if ((next !== null) !== positional) { - sound = this.restart(sound, track, next); + sound = this.restart(sound, track, next, facing); positional = next !== null; } else if (next !== null) { sound.position = next; + sound.voicePosition = next; + this.refreshSend(sound); + } + }, + setFacing: (next) => { + const turned = facingOf(next); + if (!this.voices.has(sound) || sameFacing(turned, facing)) { + return; + } + facing = turned; + if (positional) { + sound.threeDOptions = facingOptions(facing); } }, setRoute: (route) => { @@ -101,19 +181,21 @@ export class MediaVoices { * fixes a stream sound's panning mode when the sound is created. With a * position it is an HRTF point source. Without one it is a stereo source at * centre pan: the voice has no 3D panner at all, so it has no position, no - * distance model and no direction, wherever the listener stands or faces. + * distance model, no cone and no direction, wherever the listener stands or + * faces. */ - private start(track: MediaStreamTrack, position: VoicePosition): VoiceSound { + private start(track: MediaStreamTrack, position: VoicePosition, facing: VoiceFacing): VoiceSound { track.enabled = true; const sound: VoiceSound = this.cacophony.createMediaStreamSound(new MediaStream([track]), { panType: position ? 'HRTF' : 'stereo', stopTracksOnStop: false, }); if (position) { - sound.threeDOptions = VOICE_PANNER; + sound.threeDOptions = { ...VOICE_PANNER, ...facingOptions(facing) }; sound.position = position; - // The first pose is immediate; once playing, moves ramp, which de-zippers - // the per-frame steps the position tweener delivers. + sound.voicePosition = position; + // The first pose is immediate; once playing, moves and turns ramp, which + // de-zippers the per-frame steps the position and facing tweeners deliver. sound.spatialSmoothingTau = SPATIAL_PARAM_TAU_S; } sound.play(); @@ -127,11 +209,16 @@ export class MediaVoices { * track keeps running and the new sound takes over the route the old one * had, or still wanted. */ - private restart(old: VoiceSound, track: MediaStreamTrack, position: VoicePosition): VoiceSound { + private restart( + old: VoiceSound, + track: MediaStreamTrack, + position: VoicePosition, + facing: VoiceFacing, + ): VoiceSound { const { namedChain, namedSend } = old; this.voices.delete(old); old.cleanup(); - const sound = this.start(track, position); + const sound = this.start(track, position, facing); this.route(sound, namedChain, namedSend); return sound; } @@ -158,6 +245,13 @@ export class MediaVoices { } } + /** The listener moved: every positional voice's distance gain changed with it. */ + listenerMoved(): void { + for (const voice of this.voices) { + this.refreshSend(voice); + } + } + private route(voice: VoiceSound, chain: string | undefined, send: number | undefined): void { if (chain && !this.chains.hasChain(chain)) { // Not defined (yet): play dry and remember it for chainCreated. @@ -169,12 +263,36 @@ export class MediaVoices { } return; } - const error = routeNamedChain(voice, chain, send, this.master()); + const error = routeNamedChain(voice, chain, send, this.master(), this.sendGain(voice, send)); if (error) { console.warn(`Voice audio: chain '${chain}' unavailable; playing dry`, error); } } + /** + * The gain for a voice's send to its chain. A positional voice's distance + * rolloff is inside its panner, which is ahead of the gain node Cacophony + * taps a send from: left alone, a voice's reverb would fall with distance as + * fast as the voice itself. The panner's distance gain cannot be read back, + * so it is computed here from the same profile and the same positions the + * panner is given, and divided out of the send, as for a media sound. + * A non-positional voice has no distance gain. + */ + private sendGain(voice: VoiceSound, send: number | undefined): number | undefined { + if (send === undefined || !voice.voicePosition) { + return send; + } + const distance = distanceBetween(this.cacophony.listenerPosition, voice.voicePosition); + return distanceCompensatedSend(send, profileDistanceGain(distance, VOICE_PROFILE)); + } + + /** Bring a live send up to date with the voice's distance; a send that is not live is left alone. */ + private refreshSend(voice: VoiceSound): void { + if (voice.chainRouted && voice.namedSend !== undefined) { + this.route(voice, voice.namedChain, voice.namedSend); + } + } + private master() { return this.cacophony.getBus('master'); } diff --git a/src/audio/distanceModel.test.ts b/src/audio/distanceModel.test.ts index 488b5037..0365f18b 100644 --- a/src/audio/distanceModel.test.ts +++ b/src/audio/distanceModel.test.ts @@ -3,8 +3,11 @@ import { describe, expect, it } from 'vitest'; import { DEFAULT_SPATIAL_PROFILE, distanceBetween, + distanceCompensatedSend, inverseDistanceGain, + MAX_SEND_DISTANCE_BOOST, profileDistanceGain, + profilePannerDistance, SPATIAL_DISTANCE_MODEL, type SpatialProfile, } from './distanceModel'; @@ -100,3 +103,55 @@ describe('distanceBetween', () => { expect(distanceBetween([1, 2], [1, 2, 3])).toBe(0); }); }); + +describe('profilePannerDistance', () => { + it('gives the panner the default profile as its own inverse model', () => { + expect(profilePannerDistance(DEFAULT_SPATIAL_PROFILE)).toEqual({ + distanceModel: 'inverse', + refDistance: DEFAULT_SPATIAL_PROFILE.refDistance, + rolloffFactor: DEFAULT_SPATIAL_PROFILE.rolloff, + maxDistance: DEFAULT_SPATIAL_PROFILE.maxDistance, + }); + }); + + it('maps a linear profile to the linear model and none to no rolloff', () => { + const profile = { ...DEFAULT_SPATIAL_PROFILE, refDistance: 2, maxDistance: 40, rolloff: 0.5 }; + expect(profilePannerDistance({ ...profile, model: 'linear' })).toEqual({ + distanceModel: 'linear', + refDistance: 2, + rolloffFactor: 0.5, + maxDistance: 40, + }); + expect(profilePannerDistance({ ...profile, model: 'none' })).toMatchObject({ + distanceModel: 'inverse', + rolloffFactor: 0, + }); + }); +}); + +describe('distanceCompensatedSend', () => { + it('undoes the distance gain, so gain x send is the requested send', () => { + expect(distanceCompensatedSend(0.3, 0.25)).toBeCloseTo(1.2, 12); + expect(0.25 * distanceCompensatedSend(0.3, 0.25)).toBeCloseTo(0.3, 12); + expect(distanceCompensatedSend(0.3, 0.5)).toBeCloseTo(0.6, 12); + }); + + it('leaves the send alone at full distance gain', () => { + expect(distanceCompensatedSend(0.3, 1)).toBe(0.3); + expect(distanceCompensatedSend(0, 0.25)).toBe(0); + }); + + it('caps the boost, so the reverb falls with the direct sound beyond the cap', () => { + const atCap = 1 / MAX_SEND_DISTANCE_BOOST; + expect(distanceCompensatedSend(0.3, atCap)).toBeCloseTo(0.3 * MAX_SEND_DISTANCE_BOOST, 12); + expect(distanceCompensatedSend(0.3, atCap / 10)).toBeCloseTo(0.3 * MAX_SEND_DISTANCE_BOOST, 12); + // The floor of the default curve (10 km) would otherwise ask for 10 000 x. + const floor = profileDistanceGain(Number.MAX_VALUE, DEFAULT_SPATIAL_PROFILE); + expect(distanceCompensatedSend(1, floor)).toBe(MAX_SEND_DISTANCE_BOOST); + }); + + it('stays finite where a linear curve reaches silence', () => { + expect(distanceCompensatedSend(0.3, 0)).toBeCloseTo(0.3 * MAX_SEND_DISTANCE_BOOST, 12); + expect(distanceCompensatedSend(0.3, Number.NaN)).toBeCloseTo(0.3 * MAX_SEND_DISTANCE_BOOST, 12); + }); +}); diff --git a/src/audio/distanceModel.ts b/src/audio/distanceModel.ts index bad25bf4..383d4e49 100644 --- a/src/audio/distanceModel.ts +++ b/src/audio/distanceModel.ts @@ -89,6 +89,50 @@ export function profileDistanceGain(distance: number, profile: SpatialProfile): return refDistance / (refDistance + rolloff * (clamped - refDistance)); } +/** + * PannerNode distance parameters under which the panner's own rolloff is + * {@link profileDistanceGain} for the profile: Web Audio's `inverse` and + * `linear` models are the same formulas, with the same clamp at maxDistance. + * `none` is the inverse model with no rolloff. (Web Audio limits a linear + * rolloff to 0..1; a profile outside that range has no native equivalent.) + */ +export function profilePannerDistance(profile: SpatialProfile): { + distanceModel: 'inverse' | 'linear'; + refDistance: number; + rolloffFactor: number; + maxDistance: number; +} { + return { + distanceModel: profile.model === 'linear' ? 'linear' : 'inverse', + refDistance: profile.refDistance, + rolloffFactor: profile.model === 'none' ? 0 : profile.rolloff, + maxDistance: profile.maxDistance, + }; +} + +/** + * The most a reverb send is raised to undo a source's distance gain (32 x is + * +30 dB). The engine takes a send after the source's gain and accepts any + * finite send gain, so it sets no limit of its own, and the curves give none + * that is usable: the default one floors at 1 / 10 000 and a linear one + * reaches 0, where the tap is silent and no boost can recover it. So this is a + * choice: under the default curve a room's reverberant level is even out to + * 32 m, and past that it falls with the direct sound. It is a cap on the + * boost, not on the send, so that distance is the same whatever the send. + */ +export const MAX_SEND_DISTANCE_BOOST = 32; + +/** + * The gain to give a send that is tapped after `distanceGain`, so that what + * reaches the reverb is `send` times the undistanced sound: send / distanceGain, + * with the boost capped at {@link MAX_SEND_DISTANCE_BOOST}. The direct sound + * still falls with distance, so the direct-to-reverb ratio does too. + */ +export function distanceCompensatedSend(send: number, distanceGain: number): number { + const boost = distanceGain > 0 ? Math.min(1 / distanceGain, MAX_SEND_DISTANCE_BOOST) : MAX_SEND_DISTANCE_BOOST; + return send * boost; +} + /** Euclidean distance between two 3D positions; missing/short vectors → 0 (co-located). */ export function distanceBetween( a?: readonly number[] | null, diff --git a/src/audio/effects/EffectChain.ts b/src/audio/effects/EffectChain.ts index 149b377f..fd881ff8 100644 --- a/src/audio/effects/EffectChain.ts +++ b/src/audio/effects/EffectChain.ts @@ -60,6 +60,8 @@ export class EffectChain { private downstream: Bus | null = null; /** Aux-send target fed in parallel with the primary output, if any. */ private sendTarget: Bus | null = null; + /** The gain of the feed into {@link sendTarget}. */ + private sendGain = 1; private constructor(cacophony: Cacophony, id: string | null, bus: Bus, mode: ChainMode) { this.cacophony = cacophony; @@ -127,6 +129,10 @@ export class EffectChain { if (this.bus.destroyed) { return; } + if (target && this.sendTarget === target && this.sendGain === gain) { + // Already fed at this level: the feed is refreshed on every listener step. + return; + } if (this.sendTarget && this.sendTarget !== target) { this.bus.disconnect(this.sendTarget); this.sendTarget = null; @@ -134,6 +140,7 @@ export class EffectChain { if (target) { this.bus.connect(target, gain); this.sendTarget = target; + this.sendGain = gain; } } diff --git a/src/audio/namedRoute.ts b/src/audio/namedRoute.ts index 0a0a00d9..4f31d8ce 100644 --- a/src/audio/namedRoute.ts +++ b/src/audio/namedRoute.ts @@ -7,9 +7,15 @@ import type { Bus } from 'cacophony'; export interface NamedRouteState { /** Named chain the source routes to (primary, or aux when {@link namedSend} is set). */ namedChain?: string; + /** The send the server asked for. */ namedSend?: number; /** Whether {@link namedChain} is currently applied as the source's own route. */ chainRouted?: boolean; + /** + * The gain the live aux send has in the engine: {@link namedSend}, raised to + * undo the source's distance gain when the send is tapped after it. + */ + sendGain?: number; } export interface NamedRoutable extends NamedRouteState { @@ -22,6 +28,7 @@ export function clearNamedRoute(source: NamedRoutable, master: Bus | undefined): const chain = source.namedChain; const routed = source.chainRouted; source.chainRouted = false; + source.sendGain = undefined; if (!routed || !chain) { return; } @@ -39,6 +46,9 @@ export function clearNamedRoute(source: NamedRoutable, master: Bus | undefined): /** * Route the source to `chain` (primary, or an aux send at `send`), first * undoing a different named route. A no-op when that route is already live. + * `compensatedSend` is the gain the engine gives the send, when that is not + * `send` itself (see {@link NamedRouteState.sendGain}); a live aux send to the same + * chain is re-gained in place, never removed and added again. * Returns the error when the chain cannot be routed to; the source then plays * dry and keeps the route it wants, so a later call can apply it. */ @@ -47,23 +57,33 @@ export function routeNamedChain( chain: string | undefined, send: number | undefined, master: Bus | undefined, + compensatedSend: number | undefined = send, ): unknown { - if (source.chainRouted && source.namedChain === chain && source.namedSend === send) { + // A primary route has no send gain, whatever the caller computed. + const sendGain = send === undefined ? undefined : compensatedSend; + const live = source.chainRouted && source.namedChain === chain; + if (live && source.namedSend === send && source.sendGain === sendGain) { return undefined; } - clearNamedRoute(source, master); + // The engine re-gains an existing send; only a different chain, or a change + // between a primary route and an aux send, needs the old route undone. + const regain = live && source.namedSend !== undefined && send !== undefined; + if (!regain) { + clearNamedRoute(source, master); + } source.namedChain = chain; source.namedSend = send; if (!chain) { return undefined; } try { - if (send !== undefined) { - source.routeTo(chain, send); + if (sendGain !== undefined) { + source.routeTo(chain, sendGain); } else { source.routeTo(chain); } source.chainRouted = true; + source.sendGain = sendGain; return undefined; } catch (error) { return error ?? new Error(`chain '${chain}' unavailable`); diff --git a/src/components/audioChat.tsx b/src/components/audioChat.tsx index 33b044d5..09755af2 100644 --- a/src/components/audioChat.tsx +++ b/src/components/audioChat.tsx @@ -45,6 +45,8 @@ const SpatialLiveKitAudio: React.FC = ({ client, ch bridgeRef.current = new LiveKitSpatialAudioBridge( client.media, (participantId) => useSpatialStore.getState().spatialEntities[participantId]?.position, + // Already in Web Audio axes: Client.Spatial converts and tweens it into the store. + (participantId) => useSpatialStore.getState().spatialEntities[participantId]?.forward, ); } diff --git a/src/gmcp/Client/Media.test.ts b/src/gmcp/Client/Media.test.ts index 289da5cb..7fa9e1cd 100644 --- a/src/gmcp/Client/Media.test.ts +++ b/src/gmcp/Client/Media.test.ts @@ -24,6 +24,7 @@ vi.mock('../../audio/PositionalFoaRenderer', () => ({ }, })); +import { MAX_SEND_DISTANCE_BOOST } from '../../audio/distanceModel'; import { MEDIA_SEEK_TOLERANCE_MS, MediaService } from '../../audio/MediaService'; import { MediaPayloadError } from '../../audio/mediaPayloads'; import { VectorTweener } from '../../audio/vectorTween'; @@ -1996,6 +1997,205 @@ describe('GMCPClientMedia', () => { }); }); + describe('reverb send independent of distance', () => { + // Mongoose north 4 m: distance 4 from a listener at the origin, so the + // default inverse curve gives a distance gain of 0.25. + const fire = { + key: 'fire', + name: 'fire.ogg', + type: 'sound', + volume: 50, + is3d: true, + position: [0, 4, 0], + chain: 'room', + send: 0.3, + }; + + async function playFire(extra: Record = {}) { + const sound = createMockSound('https://media.example/fire.ogg'); + mockCreateSound.mockResolvedValue(sound); + await handler.handlePlay({ ...fire, ...extra } as GMCPMessageClientMediaPlay); + return sound; + } + + function lastSend(sound: MockSound): unknown[] { + return sound.routeTo.mock.calls[sound.routeTo.mock.calls.length - 1]; + } + + it('sends a positional sound at send / distance gain, leaving the direct level alone', async () => { + const sound = await playFire(); + + expect(sound.routeTo).toHaveBeenCalledOnce(); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(1.2, 9)]); + // volume x distance gain x effective send = volume x send + expect(sound.volume).toBeCloseTo(0.5 * 0.25, 9); + expect(sound.volume * (lastSend(sound)[1] as number)).toBeCloseTo(0.5 * 0.3, 9); + }); + + it('re-gains the send in place when the listener moves', async () => { + const sound = await playFire(); + + // 2 m from the source: distance gain 0.5. + client.media.setListenerPosition([0, 0, 2]); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.6, 9)]); + expect(sound.volume).toBeCloseTo(0.5 * 0.5, 9); + + // On top of it: no attenuation, so the send is the wire send. + client.media.setListenerPosition([0, 0, 4]); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.3, 9)]); + + expect(sound.removeSend).not.toHaveBeenCalled(); + expect(sound.voice.play).toHaveBeenCalledOnce(); + }); + + it('does not call the engine again when the listener moves without changing the distance', async () => { + const sound = await playFire(); + sound.routeTo.mockClear(); + + // Still 4 m away, on the other side. + client.media.setListenerPosition([0, 0, 8]); + expect(sound.routeTo).not.toHaveBeenCalled(); + }); + + it('follows the source as it glides', async () => { + const sound = await playFire(); + + handler.handleUpdate({ key: 'fire', position: [0, 2, 0] } as GMCPMessageClientMediaUpdate); + client.stepMotion(10_000); + + expect(sound.volume).toBeCloseTo(0.5 * 0.5, 9); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.6, 9)]); + }); + + it('caps the boost for a very distant sound', async () => { + const sound = await playFire(); + + client.media.setListenerPosition([0, 0, 1004]); + expect(sound.volume).toBeCloseTo(0.5 / 1000, 9); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.3 * MAX_SEND_DISTANCE_BOOST, 9)]); + }); + + it('stays finite where a linear profile reaches silence', async () => { + const sound = await playFire({ + spatial: { + model: 'linear', + refDistance: 1, + maxDistance: 4, + rolloff: 1, + coneInnerAngle: 360, + coneOuterAngle: 360, + coneOuterGain: 1, + }, + }); + + expect(sound.volume).toBe(0); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.3 * MAX_SEND_DISTANCE_BOOST, 9)]); + }); + + it('recomputes when an Update changes the send, without dropping the send', async () => { + const sound = await playFire(); + + handler.handleUpdate({ key: 'fire', send: 0.5 } as GMCPMessageClientMediaUpdate); + + expect(lastSend(sound)).toEqual(['room', expect.closeTo(2, 9)]); + expect(sound.removeSend).not.toHaveBeenCalled(); + }); + + it('recomputes when an Update changes the spatial profile', async () => { + const sound = await playFire(); + + handler.handleUpdate({ + key: 'fire', + spatial: { + model: 'inverse', + refDistance: 2, + maxDistance: 50, + rolloff: 1, + coneInnerAngle: 360, + coneOuterAngle: 360, + coneOuterGain: 1, + }, + } as GMCPMessageClientMediaUpdate); + + // 2 / (2 + (4 - 2)) = 0.5 + expect(sound.volume).toBeCloseTo(0.5 * 0.5, 9); + expect(lastSend(sound)).toEqual(['room', expect.closeTo(0.6, 9)]); + }); + + it('carries the compensated send to a new chain', async () => { + const sound = await playFire(); + + handler.handleUpdate({ key: 'fire', chain: 'hall' } as GMCPMessageClientMediaUpdate); + + expect(sound.removeSend).toHaveBeenCalledWith('room'); + expect(lastSend(sound)).toEqual(['hall', expect.closeTo(1.2, 9)]); + }); + + it('leaves a non-positional sound at its wire send, wherever the listener goes', async () => { + const sound = await playFire({ is3d: false, position: undefined }); + + expect(sound.routeTo).toHaveBeenCalledOnce(); + expect(sound.routeTo).toHaveBeenCalledWith('room', 0.3); + + client.media.setListenerPosition([0, 0, 50]); + handler.handleUpdate({ key: 'fire', volume: 25 } as GMCPMessageClientMediaUpdate); + expect(sound.routeTo).toHaveBeenCalledOnce(); + expect(sound.volume).toBe(0.25); + }); + + it('makes no send call for a sound with no chain', async () => { + const sound = await playFire({ chain: undefined }); + + client.media.setListenerPosition([0, 0, 2]); + handler.handleUpdate({ key: 'fire', position: [0, 9, 0] } as GMCPMessageClientMediaUpdate); + client.stepMotion(10_000); + + expect(sound.routeTo).not.toHaveBeenCalled(); + expect(sound.removeSend).not.toHaveBeenCalled(); + }); + + it('leaves a primary (send-less) route alone', async () => { + const sound = await playFire({ send: undefined }); + + client.media.setListenerPosition([0, 0, 2]); + + expect(sound.routeTo).toHaveBeenCalledOnce(); + expect(sound.routeTo).toHaveBeenCalledWith('room'); + }); + + it('does not retry a send the engine refused each time the listener moves', async () => { + const consoleWarn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const sound = createMockSound('https://media.example/fire.ogg'); + sound.routeTo.mockImplementation(() => { + throw new Error("No bus registered with name 'room'"); + }); + mockCreateSound.mockResolvedValue(sound); + await handler.handlePlay(fire as GMCPMessageClientMediaPlay); + expect(sound.routeTo).toHaveBeenCalledOnce(); + + client.media.setListenerPosition([0, 0, 2]); + expect(sound.routeTo).toHaveBeenCalledOnce(); + consoleWarn.mockRestore(); + }); + + it('compensates the aux feed of an inline effect chain the same way', async () => { + await client.media.setChain({ id: 'room', effects: [{ type: 'reverb' }] }); + const room = client.effectBuses.created.room; + const sound = await playFire({ + effects: [{ id: 'muffle', type: 'lowpass', params: { frequency: 400 } }], + }); + const inline = client.effectBuses.anon[0]; + + expect(sound.routeTo).toHaveBeenCalledWith(inline); + expect(inline.connect).toHaveBeenLastCalledWith(room, expect.closeTo(1.2, 9)); + + client.media.setListenerPosition([0, 0, 2]); + expect(inline.connect).toHaveBeenLastCalledWith(room, expect.closeTo(0.6, 9)); + // The dry path stays on master throughout. + expect(inline.disconnect).not.toHaveBeenCalled(); + }); + }); + describe('per-voice occlusion', () => { const tone = { key: 's4894767', @@ -2767,12 +2967,15 @@ describe('GMCPClientMedia', () => { second.voice.play.mock.invocationCallOrder[0], ); expect(handler.sounds.fire.mediaVolume).toBe(0.2); - expect(second.routeTo).toHaveBeenLastCalledWith('workshop', 0.3); + expect(handler.sounds.fire.namedSend).toBe(0.3); expect(second.loop).toHaveBeenCalledWith('infinite'); if (mode === 'HRTF') { expect(second.position).toEqual([-2, 0, 0]); + // 2 m away: distance gain 0.5, which the send (tapped after it) undoes. + expect(second.routeTo).toHaveBeenLastCalledWith('workshop', expect.closeTo(0.6, 9)); } else { expect(second.volume).toBe(0.2); + expect(second.routeTo).toHaveBeenLastCalledWith('workshop', 0.3); } });