Distance cues: level reverb sends and facing voices

src/audio/LiveKitSpatialAudioBridge.test.ts

@@ -1,15 +1,17 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
+import { useSpatialStore } from "../stores/spatialStore";
 import { LiveKitSpatialAudioBridge } from "./LiveKitSpatialAudioBridge";
 
 function createMedia() {
   const voices: Array<{
     detach: ReturnType<typeof vi.fn>;
+    setFacing: ReturnType<typeof vi.fn>;
     setPosition: ReturnType<typeof vi.fn>;
     setRoute: ReturnType<typeof vi.fn>;
   }> = [];
   const media = {
     attachVoice: vi.fn(() => {
-      const voice = { detach: vi.fn(), setPosition: vi.fn(), setRoute: vi.fn() };
+      const voice = { detach: vi.fn(), setFacing: vi.fn(), setPosition: vi.fn(), setRoute: vi.fn() };
       voices.push(voice);
       return voice;
     }),
@@ -33,7 +35,8 @@ describe("LiveKitSpatialAudioBridge", () => {
 
     bridge.attachParticipantTrack("player-2", remoteTrack);
 
-    expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3]);
+    // No facing lookup was given: the voice has none.
+    expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3], null);
   });
 
   it("attaches a participant who is not in the scene as a non-positional voice", () => {
@@ -44,7 +47,7 @@ describe("LiveKitSpatialAudioBridge", () => {
     bridge.attachParticipantTrack("player-2", remoteTrack);
 
     // A phone-call partner has no place in this room: never the room origin.
-    expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, null);
+    expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, null, null);
   });
 
   it("makes the voice positional when its entity appears, and non-positional when it leaves", () => {
@@ -127,4 +130,78 @@ describe("LiveKitSpatialAudioBridge", () => {
     expect(voices[1].detach).toHaveBeenCalledOnce();
     expect(voices[0].detach).toHaveBeenCalledOnce();
   });
+
+  describe("the speaker's facing", () => {
+    it("attaches the voice facing the way its entity does", () => {
+      const { media } = createMedia();
+      const remoteTrack = track("a");
+      const bridge = new LiveKitSpatialAudioBridge(
+        media,
+        () => [1, 2, 3],
+        () => [0, 0, -1],
+      );
+
+      bridge.attachParticipantTrack("player-2", remoteTrack);
+
+      expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3], [0, 0, -1]);
+    });
+
+    it("attaches an entity with no forward, and a bridge given no facing lookup, with no facing", () => {
+      const { media } = createMedia();
+      const remoteTrack = track("a");
+
+      new LiveKitSpatialAudioBridge(
+        media,
+        () => [1, 2, 3],
+        () => undefined,
+      ).attachParticipantTrack("player-2", remoteTrack);
+      new LiveKitSpatialAudioBridge(media, () => [1, 2, 3]).attachParticipantTrack("player-3", remoteTrack);
+
+      expect(media.attachVoice).toHaveBeenNthCalledWith(1, remoteTrack, [1, 2, 3], null);
+      expect(media.attachVoice).toHaveBeenNthCalledWith(2, remoteTrack, [1, 2, 3], null);
+    });
+
+    it("turns the voice on sync, and drops the facing when the forward goes", () => {
+      const { media, voices } = createMedia();
+      const facings: Record<string, [number, number, number] | undefined> = { "player-2": [0, 0, -1] };
+      const bridge = new LiveKitSpatialAudioBridge(
+        media,
+        () => [1, 2, 3],
+        (participantId) => facings[participantId],
+      );
+      bridge.attachParticipantTrack("player-2", track("a"));
+
+      facings["player-2"] = [1, 0, 0];
+      bridge.syncAll();
+      expect(voices[0].setFacing).toHaveBeenLastCalledWith([1, 0, 0]);
+
+      delete facings["player-2"];
+      bridge.syncParticipant("player-2");
+      expect(voices[0].setFacing).toHaveBeenLastCalledWith(null);
+    });
+
+    it("follows the entity through the spatial store, as the audio chat subscribes it", () => {
+      const store = useSpatialStore;
+      store.getState().reset();
+      store.getState().enterEntity({ id: "player-2", position: [1, 2, 3], forward: [0, 0, -1] });
+      const { media, voices } = createMedia();
+      const bridge = new LiveKitSpatialAudioBridge(
+        media,
+        (participantId) => store.getState().spatialEntities[participantId]?.position,
+        (participantId) => store.getState().spatialEntities[participantId]?.forward,
+      );
+      const unsubscribe = store.subscribe(() => bridge.syncAll());
+      const remoteTrack = track("a");
+      bridge.attachParticipantTrack("player-2", remoteTrack);
+      expect(media.attachVoice).toHaveBeenCalledWith(remoteTrack, [1, 2, 3], [0, 0, -1]);
+
+      // A step of the turn Client.Spatial.EntityMove tweens into the store.
+      store.getState().patchEntity("player-2", { forward: [1, 0, 0] });
+      expect(voices[0].setFacing).toHaveBeenLastCalledWith([1, 0, 0]);
+      expect(voices[0].setPosition).toHaveBeenLastCalledWith([1, 2, 3]);
+
+      unsubscribe();
+      store.getState().reset();
+    });
+  });
 });

src/audio/LiveKitSpatialAudioBridge.ts

@@ -1,11 +1,14 @@
 import type { Position } from "cacophony";
 
 import type { MediaService } from "./MediaService";
-import type { MediaVoice, VoicePosition, VoiceRoute } from "./MediaVoices";
+import type { MediaVoice, VoiceFacing, VoicePosition, VoiceRoute } from "./MediaVoices";
 
 /** The participant's position as an entity in the current scene; null or undefined when not in it. */
 export type SpatialPositionLookup = (participantId: string) => Position | null | undefined;
 
+/** The way the participant's entity faces (its `forward`); null or undefined when it has none. */
+export type SpatialFacingLookup = (participantId: string) => Position | null | undefined;
+
 type VoiceMedia = Pick<MediaService, "attachVoice">;
 
 interface SpatialAudioEntry {
@@ -15,8 +18,8 @@ interface SpatialAudioEntry {
 
 /**
  * One LiveKit room's remote participants, as voices in the shared audio graph:
- * positioned at their entity while it is in the current scene, non-positional
- * otherwise.
+ * positioned at their entity, and facing the way it faces, while it is in the
+ * current scene; non-positional otherwise.
  */
 export class LiveKitSpatialAudioBridge {
   private readonly entries = new Map<string, SpatialAudioEntry>();
@@ -25,6 +28,7 @@ export class LiveKitSpatialAudioBridge {
   constructor(
     private readonly media: VoiceMedia,
     private readonly lookupPosition: SpatialPositionLookup,
+    private readonly lookupFacing: SpatialFacingLookup = () => null,
   ) {}
 
   attachParticipantTrack(participantId: string, track: MediaStreamTrack): void {
@@ -37,7 +41,11 @@ export class LiveKitSpatialAudioBridge {
     this.detachParticipant(participantId);
 
     // A first placement snaps, so a new voice does not fly in from the origin.
-    const voice = this.media.attachVoice(track, this.positionFor(participantId));
+    const voice = this.media.attachVoice(
+      track,
+      this.positionFor(participantId),
+      this.facingFor(participantId),
+    );
     voice.setRoute(this.route);
     this.entries.set(participantId, { track, voice });
   }
@@ -50,8 +58,14 @@ export class LiveKitSpatialAudioBridge {
     }
   }
 
+  /** Bring the participant's voice to where its entity is now, and the way it faces. */
   syncParticipant(participantId: string): void {
-    this.entries.get(participantId)?.voice.setPosition(this.positionFor(participantId));
+    const voice = this.entries.get(participantId)?.voice;
+    if (!voice) {
+      return;
+    }
+    voice.setPosition(this.positionFor(participantId));
+    voice.setFacing(this.facingFor(participantId));
   }
 
   syncAll(): void {
@@ -93,4 +107,9 @@ export class LiveKitSpatialAudioBridge {
   private positionFor(participantId: string): VoicePosition {
     return this.lookupPosition(participantId) ?? null;
   }
+
+  /** The way the participant's entity faces, or null: a voice with no facing has no cone. */
+  private facingFor(participantId: string): VoiceFacing {
+    return this.lookupFacing(participantId) ?? null;
+  }
 }

src/audio/MediaService.ts

@@ -21,10 +21,16 @@ import type { PositionalFoaRenderer } from './PositionalFoaRenderer';
 import {
   DEFAULT_SPATIAL_PROFILE,
   distanceBetween,
+  distanceCompensatedSend,
   profileDistanceGain,
   type SpatialProfile,
 } from './distanceModel';
-import { type MediaVoice, MediaVoices, type VoicePosition } from './MediaVoices';
+import {
+  type MediaVoice,
+  MediaVoices,
+  type VoiceFacing,
+  type VoicePosition,
+} from './MediaVoices';
 import { clearNamedRoute, type NamedRouteState, routeNamedChain } from './namedRoute';
 import { hasOcclusion, OCCLUSION_GLIDE_MS } from './occlusion';
 import { VectorTweener } from './vectorTween';
@@ -351,6 +357,7 @@ export class MediaService {
         this.updatePositionalSpatial(sound);
         this.applyLevels(sound);
       }
+      this.voices.listenerMoved();
     }
   }
 
@@ -385,20 +392,65 @@ export class MediaService {
     sound.ambisonicRenderer.setDistanceGain(this.distanceGain(sound));
   }
 
+  /**
+   * The distance gain that is part of the sound's own gain: its profile's, for
+   * an HRTF point source, and 1 for every other sound (non-positional sounds
+   * have none; ambisonic routes apply distance in their renderer instead).
+   */
+  private levelDistanceGain(sound: ExtendedSound): number {
+    return sound.pointSource && sound.upmix !== 'ambisonic' ? this.distanceGain(sound) : 1;
+  }
+
   /**
    * Set the sound's gain: wire volume × 10^(gainDb/20) × (for an HRTF point
    * source) its profile distance gain. This is the one stage where distance
    * falloff is applied on the HRTF route; the panner's rolloff is 0. Ambisonic
    * routes apply distance in their renderer instead.
+   *
+   * The engine taps a send after this gain, so the send to the sound's chain
+   * is re-gained here too: every change of the distance gain (the listener or
+   * the source moves, the profile changes) comes through this method.
    */
   private applyLevels(sound: ExtendedSound): void {
     if (this.cleanedSounds.has(sound)) {
       return;
     }
     const volume = sound.mediaVolume ?? 1;
     const gain = sound.gainDb ? 10 ** (sound.gainDb / 20) : 1;
-    const distance = sound.pointSource && sound.upmix !== 'ambisonic' ? this.distanceGain(sound) : 1;
-    sound.volume = volume * gain * distance;
+    sound.volume = volume * gain * this.levelDistanceGain(sound);
+    this.refreshSend(sound);
+  }
+
+  /**
+   * The gain for the sound's send to its named chain. Cacophony takes a send
+   * from the voice's output, after its gain, so the wire `send` alone would
+   * make the reverberant level fall with distance exactly as the direct level
+   * does. A room's reverberant field is roughly even, and the direct-to-reverb
+   * ratio is how a listener hears distance: so the distance gain is divided
+   * back out of the send ({@link distanceCompensatedSend}) and the chain gets
+   * volume × send wherever the source is. Occlusion sits before the tap and
+   * still dims direct and reverb alike.
+   */
+  private sendGain(sound: ExtendedSound, send: number | undefined): number | undefined {
+    return send === undefined
+      ? undefined
+      : distanceCompensatedSend(send, this.levelDistanceGain(sound));
+  }
+
+  /**
+   * Bring a live send up to date with the sound's distance gain. Only a send
+   * that is already routed is touched: one the engine refused is not retried
+   * on every listener step.
+   */
+  private refreshSend(sound: ExtendedSound): void {
+    if (sound.namedSend === undefined) {
+      return;
+    }
+    if (sound.effectChain) {
+      this.pointInlineChain(sound.effectChain, sound.namedChain, this.sendGain(sound, sound.namedSend));
+    } else if (sound.chainRouted) {
+      this.routeNamedChain(sound, sound.namedChain, sound.namedSend);
+    }
   }
 
   /** HRTF panner settings for a point source: position and cone only, no native rolloff. */
@@ -437,11 +489,16 @@ export class MediaService {
 
   /**
    * Put a live voice track (a LiveKit participant) in the graph: an HRTF point
-   * source at `position`, or non-positional when it is null. It is not a media
-   * key: Client.Media.Stop never touches it.
+   * source at `position` facing `forward` (no facing when that is null), or
+   * non-positional when the position is null. It is not a media key:
+   * Client.Media.Stop never touches it.
    */
-  attachVoice(track: MediaStreamTrack, position: VoicePosition): MediaVoice {
-    return this.voices.attach(track, position);
+  attachVoice(
+    track: MediaStreamTrack,
+    position: VoicePosition,
+    forward: VoiceFacing = null,
+  ): MediaVoice {
+    return this.voices.attach(track, position, forward);
   }
 
   setChain(data: ClientMediaChainPayload): Promise<void> {
@@ -1130,7 +1187,13 @@ export class MediaService {
    * undoing a different named route. A no-op when that route is already live.
    */
   private routeNamedChain(sound: ExtendedSound, chain: string | undefined, send: number | undefined): void {
-    const error = routeNamedChain(sound, chain, send, this.cacophony.getBus('master'));
+    const error = routeNamedChain(
+      sound,
+      chain,
+      send,
+      this.cacophony.getBus('master'),
+      this.sendGain(sound, send),
+    );
     if (error) {
       console.warn(`Client.Media: chain '${chain}' unavailable; playing dry`, error);
       this.traceSound('routed', sound, sound.key, {
@@ -1195,7 +1258,7 @@ export class MediaService {
       // Update without effects: keep the inline chain, re-point what it feeds.
       sound.namedChain = chain;
       sound.namedSend = send;
-      this.pointInlineChain(sound.effectChain, chain, send);
+      this.pointInlineChain(sound.effectChain, chain, this.sendGain(sound, send));
       return;
     }
     this.routeNamedChain(sound, chain, send);
@@ -1237,6 +1300,8 @@ export class MediaService {
    * inline output stays on master (the dry path) and feeds the named chain at
    * the send level, exactly as chain+send behaves without inline effects.
    * Without a send, the inline chain runs in series into the named chain.
+   * `send` is the gain for that feed ({@link sendGain}): the inline bus is
+   * downstream of the sound's gain, so its feed is after the distance gain too.
    */
   private pointInlineChain(inline: EffectChain, chain: string | undefined, send: number | undefined): void {
     const target = chain ? (this.effects.getChain(chain)?.bus ?? null) : null;
@@ -1258,7 +1323,7 @@ export class MediaService {
     if (!inline) {
       return;
     }
-    this.pointInlineChain(inline, data.chain, data.send);
+    this.pointInlineChain(inline, data.chain, this.sendGain(sound, data.send));
     try {
       sound.routeTo(inline.bus);
     } catch (error) {

src/audio/MediaService.voices.test.ts

@@ -88,6 +88,21 @@ describe('MediaService voices', () => {
     expect(sound.routeTo).toHaveBeenLastCalledWith('room', 0.4);
   });
 
+  it('keeps a voice send level with the room as the listener walks away from the speaker', async () => {
+    const { media, sound } = setup();
+    await media.setChain(reverb);
+    media.setListenerPosition([0, 0, 0]);
+    const voice = media.attachVoice(track(), [0, 0, 4], [0, 0, -1]);
+    voice.setRoute({ chain: 'room', send: 0.3 });
+    // 4 m: the panner's distance gain is 0.25, ahead of the tap.
+    expect(sound.routeTo).toHaveBeenLastCalledWith('room', expect.closeTo(1.2, 9));
+    expect(sound.threeDOptions).toEqual(expect.objectContaining({ orientationZ: -1 }));
+
+    media.setListenerPosition([0, 0, 2]);
+    expect(sound.routeTo).toHaveBeenLastCalledWith('room', expect.closeTo(0.6, 9));
+    expect(sound.removeSend).not.toHaveBeenCalled();
+  });
+
   it('keeps voices through a stop-all and a reset, which only returns them to dry', async () => {
     const { media, order, sound } = setup();
     await media.setChain(reverb);