From b2445241788dd3b3fcf0430b1e211ab05974c967 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Fri, 6 Nov 2026 23:07:37 +0100 Subject: [PATCH] feat(editor): mute the recording's own audio A project setting silences the take's own sound in the preview and the export. Added tracks (voiceover, music) keep playing; when nothing else is audible the exported MP4 carries no audio stream at all. Refs #1104 --- electron/ai-edition/deep-agent/service.ts | 2 +- .../ai-edition/editor-settings-tool.test.ts | 8 +++- electron/ai-edition/editor-settings-tool.ts | 2 + src/cli/CliExportRunner.tsx | 11 +++++- src/components/ai-edition/ExportDialog.tsx | 8 +++- src/components/ai-edition/RightPanes.tsx | 11 +++++- .../ai-edition/VirtualPreview.audio.test.ts | 10 +++++ src/components/ai-edition/VirtualPreview.tsx | 9 +++-- src/components/ai-edition/v4/V4Timeline.tsx | 2 +- src/i18n/locales/ar/settings.json | 1 + src/i18n/locales/cs/settings.json | 1 + src/i18n/locales/de/settings.json | 1 + src/i18n/locales/en/settings.json | 1 + src/i18n/locales/es/settings.json | 1 + src/i18n/locales/fr/settings.json | 1 + src/i18n/locales/it/settings.json | 1 + src/i18n/locales/ja-JP/settings.json | 1 + src/i18n/locales/ko-KR/settings.json | 1 + src/i18n/locales/pt-BR/settings.json | 1 + src/i18n/locales/ru/settings.json | 1 + src/i18n/locales/tr/settings.json | 1 + src/i18n/locales/vi/settings.json | 1 + src/i18n/locales/zh-CN/settings.json | 1 + src/i18n/locales/zh-TW/settings.json | 1 + .../ai-edition/store/editorSettings.test.ts | 9 +++++ src/lib/ai-edition/store/editorSettings.ts | 6 +++ src/lib/exporter/audioExport.test.ts | 20 +++++++++- src/lib/exporter/audioExport.ts | 39 +++++++++++++++++++ src/native/sceneDescription.test.ts | 29 ++++++++++++++ src/native/sceneDescription.ts | 6 ++- 30 files changed, 175 insertions(+), 12 deletions(-) diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 67c3b7df74..7d0ada1567 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -268,7 +268,7 @@ export const TOOL_DESCRIPTIONS: Record = { setAspectRatio: "Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording, or one of the clips' own shapes (getCurrentDocument's settingsContext.originalAspectRatios). Annotations on the frame keep their place and shape. The same as setEditorSettings with aspectRatio alone.", setEditorSettings: - 'Change the project\'s look: pass only the settings to change, named as getCurrentDocument returns them under `settings`. Background: wallpaper (a bundled path from settingsContext.bundledWallpapers, a CSS colour or gradient, or a base64 PNG/JPEG data URL; never a file path or web URL; an embedded image reads back as a marker, leave the key out to keep it), wallpaperMotion (moves a gradient or image only), backgroundBlur 0–1. Frame: aspectRatio ("auto", a preset, or an originalAspectRatios token; a fixed format picked for the first time fills the frame with a window that follows the cursor, which formatFollowCursor turns off when settingsContext.formatFill is "available"), padding 0–100 %, borderRadius 0–64 px, shadowIntensity 0–1, frame (none, window, laptop, phone, monitor) and frameTheme (light, dark), motionBlurAmount 0–1, depthOfField (blurs 3D-tilted zooms only). Camera: webcamLayoutPreset, webcamMirrored, webcamReactiveZoom (shrinks on zoom), webcamMaskShape (square or rectangle, the camera\'s own proportions; an untouched roundness follows it), webcamRoundness 0–1, webcamSizePreset 15–60, webcamAnchor (a corner or an edge middle), webcamCropRegion (the framing: a square share of the picture, width = height ≥ 1/3) and webcamCropPan (where that window sits, 0–1 per axis, 0.5 centred), webcamBackgroundMode (none, transparent, blur, custom: needs settingsContext.cameraSegmentation ready), webcamBlurIntensity 0–1, webcamWallpaper (as wallpaper). Cursor, under `cursor`: show, autoHide, theme (settingsContext.cursorThemes), asArrow (the recorded cursor kinds drawn as the arrow instead), size 1.5–6, smoothing 0–1, motionBlur 0–1, clickBounce 0–2, clickImpact, clickSound (only heard when the take recorded clicks) and clickSoundGainDb −24–0, model3d. Also autoFocusAll (every zoom follows the cursor) and audioGainDb −12–12 dB, the master volume. A value out of range or a key not listed is refused with the reason, and nothing changes. The result lists what was applied, including what the editor changes with it, and `inactive` for any of those the current state leaves without effect.', + 'Change the project\'s look: pass only the settings to change, named as getCurrentDocument returns them under `settings`. Background: wallpaper (a bundled path from settingsContext.bundledWallpapers, a CSS colour or gradient, or a base64 PNG/JPEG data URL; never a file path or web URL; an embedded image reads back as a marker, leave the key out to keep it), wallpaperMotion (moves a gradient or image only), backgroundBlur 0–1. Frame: aspectRatio ("auto", a preset, or an originalAspectRatios token; a fixed format picked for the first time fills the frame with a window that follows the cursor, which formatFollowCursor turns off when settingsContext.formatFill is "available"), padding 0–100 %, borderRadius 0–64 px, shadowIntensity 0–1, frame (none, window, laptop, phone, monitor) and frameTheme (light, dark), motionBlurAmount 0–1, depthOfField (blurs 3D-tilted zooms only). Camera: webcamLayoutPreset, webcamMirrored, webcamReactiveZoom (shrinks on zoom), webcamMaskShape (square or rectangle, the camera\'s own proportions; an untouched roundness follows it), webcamRoundness 0–1, webcamSizePreset 15–60, webcamAnchor (a corner or an edge middle), webcamCropRegion (the framing: a square share of the picture, width = height ≥ 1/3) and webcamCropPan (where that window sits, 0–1 per axis, 0.5 centred), webcamBackgroundMode (none, transparent, blur, custom: needs settingsContext.cameraSegmentation ready), webcamBlurIntensity 0–1, webcamWallpaper (as wallpaper). Cursor, under `cursor`: show, autoHide, theme (settingsContext.cursorThemes), asArrow (the recorded cursor kinds drawn as the arrow instead), size 1.5–6, smoothing 0–1, motionBlur 0–1, clickBounce 0–2, clickImpact, clickSound (only heard when the take recorded clicks) and clickSoundGainDb −24–0, model3d. Also autoFocusAll (every zoom follows the cursor), audioGainDb −12–12 dB, the master volume, and recordingMuted (silences the recording\'s own sound in the preview and the export; added audio tracks still play). A value out of range or a key not listed is refused with the reason, and nothing changes. The result lists what was applied, including what the editor changes with it, and `inactive` for any of those the current state leaves without effect.', addEffect: "Write a visual effect as a shader and pose it over a span of the edited timeline (virtual seconds), or pose one of this project's effects again (effectId). Use it for a look the settings do not offer. `effect.code` is WGSL that defines `fn effect(uv: vec2) -> vec4`, called for every pixel; uv is 0..1 with (0,0) top-left. Available inside it: `frame(uv)` samples the composed picture; `footage(p)` samples the recording itself at its own resolution at point p of its surface (the space of `fx_to_footage`), transparent outside 0..1, with no cursor or webcam on it: read it for anything that magnifies, frame() for the rest; `params.` reads each declared parameter; `ctx.time` is programme seconds; `ctx.region.x` goes 0 to 1 across the region (y its length in s, z seconds since it began); `ctx.resolution` and `ctx.aspect`; `ctx.cursor` (xy in uv, z = 1 when on screen); `ctx.clicks[0..3]`, the last four clicks, newest first (xy in uv where it happened, z seconds since, w = 1 when the slot holds one); `ctx.footage` and `ctx.camera`, the recording's and the webcam's rects (x, y, w, h in uv; camera zero-sized when absent) — all already placed through zoom and layout; `fx_to_footage(uv)` and `fx_from_footage(p)` map between output uv and the recording's own surface (0..1 across the recording) through the warp the compositor draws it with, zoom, layout and 3D camera included, and `ctx.footage_aspect` is its width over height: the effect works in whichever of the two spaces fits it; `fx_px(n)`, n pixels of a 1080p frame as a fraction of the height; `fx_hash(vec2)`, a random number in [0,1) identical on every GPU. Rules: the effect is a pure function of those inputs (no state between frames), so the preview and the export show the same pixels; reason in uv or fx_px, never raw pixels; every loop is a `for` with literal bounds (`for (var i = 0; i < 16; i++)`), at most 4096 iterations per pixel — the one hard rule. `params` declares up to 8 (recommended) parameters, each `{type: number|toggle|color|point, name, label, default}` (number also `min`, `max`); declare one only when moving it visibly changes the effect over its whole range. `target`: `image` (default: the picture under the annotations and captions), `text` (the annotations and captions alone, premultiplied alpha), `both` (everything); use text or both only when asked. One effect runs at a time: to combine two looks, write them in one shader. The result gives the cost tier, in frame() and footage() reads per pixel, with what a 1080p frame pays on a fast discrete GPU (RTX 4070 Ti) and on an integrated one (Apple M1): light ≤ 8, up to 0.06 ms and 1.3 ms; medium ≤ 32, 0.14 ms and 4.3 ms; heavy ≤ 128, 0.54 ms and 16 ms; very heavy beyond, 2 ms and 49 ms at 512 reads. It also gives a recommendation when the tier runs long: light anywhere, medium up to a minute, heavy up to 10 s, very heavy a few seconds. It is a recommendation, never a refusal. Code that does not compile is refused with the compiler's error at the line of your code: fix it and call again. Check the result with getFrames.", setEffect: diff --git a/electron/ai-edition/editor-settings-tool.test.ts b/electron/ai-edition/editor-settings-tool.test.ts index d838e95ce6..78907636bb 100644 --- a/electron/ai-edition/editor-settings-tool.test.ts +++ b/electron/ai-edition/editor-settings-tool.test.ts @@ -68,6 +68,7 @@ describe("setEditorSettings", () => { borderRadius: 12, frame: "laptop", audioGainDb: -3, + recordingMuted: true, autoFocusAll: true, cursor: { size: 2, theme: "glass-lens", show: true, asArrow: ["text"] }, }); @@ -80,6 +81,7 @@ describe("setEditorSettings", () => { borderRadius: 12, frame: "laptop", audioGainDb: -3, + recordingMuted: true, autoFocusAll: true, cursorTheme: "glass-lens", }); @@ -87,7 +89,11 @@ describe("setEditorSettings", () => { expect(settings.cursor.asArrow).toEqual(["text"]); const snapshot = read(document); - expect(snapshot.settings).toMatchObject({ padding: 20, aspectRatio: "auto" }); + expect(snapshot.settings).toMatchObject({ + padding: 20, + aspectRatio: "auto", + recordingMuted: true, + }); expect(snapshot.settings.cursor).toMatchObject({ size: 2, theme: "glass-lens" }); // What is read is what the tool takes: the whole block goes back in unchanged. expect(set(document, snapshot.settings).ok).toBe(true); diff --git a/electron/ai-edition/editor-settings-tool.ts b/electron/ai-edition/editor-settings-tool.ts index 4ed6bdf708..9b708b957e 100644 --- a/electron/ai-edition/editor-settings-tool.ts +++ b/electron/ai-edition/editor-settings-tool.ts @@ -130,6 +130,7 @@ export const setEditorSettingsArgs = z webcamWallpaper: wallpaperArg("webcamWallpaper"), webcamBlurIntensity: bounded("webcamBlurIntensity"), audioGainDb: z.number().min(-AUDIO_GAIN_DB_LIMIT).max(AUDIO_GAIN_DB_LIMIT), + recordingMuted: z.boolean(), autoFocusAll: z.boolean(), cursor: z .object({ @@ -191,6 +192,7 @@ function settingsForModel(document: AxcutDocument) { webcamWallpaper: elideDataUrl(s.webcamWallpaper), webcamBlurIntensity: s.webcamBlurIntensity, audioGainDb: s.audioGainDb, + recordingMuted: s.recordingMuted, autoFocusAll: s.autoFocusAll, cursor: { show: s.cursorShow, diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index 3f0041e512..5d58b1a0f6 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -26,7 +26,12 @@ import { isAudioExportFormat, } from "@/lib/cliContracts"; import { GIF_SIZE_PRESETS, type GifSizePreset } from "@/lib/exporter"; -import { extractAudio, isolateAudioTrack } from "@/lib/exporter/audioExport"; +import { + exportIsSilent, + extractAudio, + isolateAudioTrack, + removeAudioStream, +} from "@/lib/exporter/audioExport"; import { calculateMp4ExportSettings } from "@/lib/exporter/mp4ExportSettings"; import { outputFrameCount } from "@/lib/exporter/outputFrameCount"; import { mixVoiceoverIntoVideo } from "@/lib/exporter/voiceoverMix"; @@ -215,6 +220,10 @@ async function runExport(request: CliExportRequest): Promise { bitrate: outDims.bitrate, }); + if (!request.audioPath && exportIsSilent(axcutDocument, sceneDesc)) { + await removeAudioStream(outPath); + } + if (request.audioPath) { window.electronAPI.cliProgress({ percentage: 100, phase: "mixing-voiceover" }); const [videoResponse, audioResponse] = await Promise.all([ diff --git a/src/components/ai-edition/ExportDialog.tsx b/src/components/ai-edition/ExportDialog.tsx index a4c453c047..0e8b22328e 100644 --- a/src/components/ai-edition/ExportDialog.tsx +++ b/src/components/ai-edition/ExportDialog.tsx @@ -33,6 +33,7 @@ import { MP4_FRAME_RATES, type Mp4FrameRate, } from "@/lib/exporter"; +import { exportIsSilent, removeAudioStream } from "@/lib/exporter/audioExport"; import { calculateMp4ExportSettings, wouldUpscale } from "@/lib/exporter/mp4ExportSettings"; import { outputFrameCount } from "@/lib/exporter/outputFrameCount"; import { @@ -115,6 +116,7 @@ function revealExportedFile(filePath: string): void { * no camera — the no-webcam layout is a later step) and its source trim. */ function buildNativeClipList(document: AxcutDocument): CompositorClipInput[] { const assetById = new Map(document.assets.map((a) => [a.id, a])); + const hasAudio = !getEditorSettings(document).recordingMuted; return resolveVisibleClips(document).flatMap((clip) => { const asset = assetById.get(clip.assetId); if (!asset?.originalPath) { @@ -135,7 +137,7 @@ function buildNativeClipList(document: AxcutDocument): CompositorClipInput[] { sourceStartSec: clip.sourceStartSec, sourceEndSec, webcamOffsetSec: camera.offsetSec, - hasAudio: true, + hasAudio, }, ]; }); @@ -459,6 +461,10 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { job.id, ); if (activeExport.current !== job) return; + if (format !== "gif" && exportIsSilent(document, sceneDesc)) { + await removeAudioStream(pickedPath); + if (activeExport.current !== job) return; + } setSavedPath(pickedPath); setPhase("done"); // Reported after the export succeeded, so a cancelled or failed run never counts. diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 8bed94405e..e03038fbd2 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -3464,11 +3464,20 @@ export function AudioPane() { onCommit={() => void commit()} /> +
+ {ts("audio.muteRecording")} + void set({ recordingMuted: v })} + /> +
diff --git a/src/components/ai-edition/VirtualPreview.audio.test.ts b/src/components/ai-edition/VirtualPreview.audio.test.ts index d87e65816a..5ba4fafb31 100644 --- a/src/components/ai-edition/VirtualPreview.audio.test.ts +++ b/src/components/ai-edition/VirtualPreview.audio.test.ts @@ -112,6 +112,16 @@ describe("applyPreviewAudioSettings", () => { expect(element.volume).toBe(1); }); + it("silences a muted recording on its own gain, leaving the output gain to the tracks (#1104)", () => { + const graph = fakeGraph(); + applyPreviewAudioSettings(graph, [], 0, Number.NEGATIVE_INFINITY); + expect(graph.voice.gain.value).toBe(0); + expect(graph.gain.gain.value).toBe(1); + const element = { volume: 1 } as HTMLAudioElement; + applyPreviewAudioSettings(null, [element], 0, Number.NEGATIVE_INFINITY); + expect(element.volume).toBe(0); + }); + it("leaves the elements alone once the graph is carrying the gain", () => { // Their audio no longer reaches the default output, so `volume` would only // scale the signal a second time on its way into the node. diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsx index 03b7530bc8..f3a17fad03 100644 --- a/src/components/ai-edition/VirtualPreview.tsx +++ b/src/components/ai-edition/VirtualPreview.tsx @@ -681,9 +681,12 @@ export function VirtualPreview({ ); } }, [voicePathsKey, retryToken, activeVoicePath]); - const voiceGainDb = activeSource?.filePath - ? (loudnessGainDbByPath.get(activeSource.filePath) ?? 0) - : 0; + // A muted recording is silenced on its own gain, so the added tracks keep playing (#1104). + const voiceGainDb = settings.recordingMuted + ? Number.NEGATIVE_INFINITY + : activeSource?.filePath + ? (loudnessGainDbByPath.get(activeSource.filePath) ?? 0) + : 0; const voiceGainDbRef = useRef(voiceGainDb); voiceGainDbRef.current = voiceGainDb; diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index 79052a2498..7429932222 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -2642,7 +2642,7 @@ export function V4Timeline({ assetDurationSec={asset?.durationSec} sourceStartSec={c.sourceStartSec} sourceEndSec={c.sourceEndSec ?? c.sourceStartSec + dur} - gain={audioGainScalar(settings.audioGainDb)} + gain={settings.recordingMuted ? 0 : audioGainScalar(settings.audioGainDb)} />