Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions packages/engine/src/services/audioMixer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -703,6 +703,77 @@ describe("processCompositionAudio", () => {
expect(existsSync(workDir)).toBe(false);
});

const groupedEffects = (baseDir: string) => {
const track = (id: string, start: number) => ({
id,
src: `${id}.wav`,
start,
end: start + 1,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio" as const,
groupId: "sfx",
});
for (const id of ["hit", "whoosh", "pop"]) writeFileSync(join(baseDir, `${id}.wav`), "stub");
return [track("hit", 0), track("whoosh", 1), track("pop", 8)];
};

it("pads a group's members once after summing them, not each member to the film length", async () => {
// Per-member padding made the sub-mix cost members x film length: ~680 effects in a
// 10-minute film ran past the ffmpeg deadline.
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);

await processCompositionAudio(
groupedEffects(baseDir),
baseDir,
workDir,
join(baseDir, "out.m4a"),
600,
);

const groupMix = capturedFilterScripts.find((script) => script.includes("normalize=0"));
expect(groupMix).toContain("amix=inputs=3:duration=longest:dropout_transition=0:normalize=0");
expect(groupMix).not.toMatch(/adelay=[^;]*,apad/);
expect((groupMix?.match(/apad/g) ?? []).length).toBe(1);
expect(groupMix).toContain("apad,asetpts=N/SR/TB,atrim=0:600[out]");
});

it("names the deadline when a group sub-mix is killed for taking too long", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
runFfmpegMock.mockImplementation(async (args: string[]) =>
args.includes("-filter_complex_script")
? {
success: false,
durationMs: 300_000,
stderr: "Stream mapping:\n Stream #0:0 -> #0:0 (pcm_s16le (native))",
exitCode: 255,
terminationReason: "deadline",
}
: { success: true, durationMs: 1, stderr: "", exitCode: 0 },
);

const result = await processCompositionAudio(
groupedEffects(baseDir),
baseDir,
workDir,
join(baseDir, "out.m4a"),
600,
undefined,
{ ffmpegProcessTimeout: 300_000 },
);

expect(result.success).toBe(false);
const failure = result.failures?.find((f) => f.elementId === "sfx");
expect(failure?.reason).toBe("ffmpeg_timeout");
expect(failure?.detail).toContain("FFmpeg timed out after 300000 ms");
expect(failure?.detail).toContain("FFMPEG_PROCESS_TIMEOUT_MS");
});

it("lets an FX tail run past the clip, still bounded by the composition", async () => {
// A reverb is still decaying when the clip's own audio stops. Trimming at
// the clip boundary is what cut every tail short in the render.
Expand Down
32 changes: 20 additions & 12 deletions packages/engine/src/services/audioMixer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import {
writeUrlDownloadTelemetry,
} from "../utils/urlDownloader.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
import { describeFfmpegFailure, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
import { encodeAac } from "./aacEncode.js";
import { audioSubprocessEvidence } from "./audioFailureDiagnostic.js";
import { unwrapTemplate } from "../utils/htmlTemplate.js";
Expand Down Expand Up @@ -446,6 +446,7 @@ function downloadFailure(error: unknown, elementId: string): AudioProcessingFail
function ffmpegFailure(
stage: Extract<AudioFailureStage, "extract" | "prepare" | "mix" | "silence">,
result: RunFfmpegResult,
timeoutMs: number,
elementId?: string,
): AudioProcessingFailure {
const stderr = result.stderr ?? "";
Expand Down Expand Up @@ -486,8 +487,8 @@ function ffmpegFailure(
subprocess: audioSubprocessEvidence(result),
detail: boundedDetail(
result.error?.message
? `${formatFfmpegError(result.exitCode, stderr)}: ${result.error.message}`
: formatFfmpegError(result.exitCode, stderr),
? `${describeFfmpegFailure(result, timeoutMs)}: ${result.error.message}`
: describeFfmpegFailure(result, timeoutMs),
),
};
}
Expand Down Expand Up @@ -724,7 +725,7 @@ async function extractAudioSegment(
failure,
};
}
let failure = result.success ? undefined : ffmpegFailure(stage, result);
let failure = result.success ? undefined : ffmpegFailure(stage, result, ffmpegProcessTimeout);
const probedLinkFailedToRead =
readsUrl && (failure?.reason === "invalid_media" || failure?.reason === "ffmpeg_failed");
if (failure && probedLinkFailedToRead) {
Expand Down Expand Up @@ -780,7 +781,9 @@ async function generateSilence(
failure,
};
}
const failure = !result.success ? ffmpegFailure("silence", result) : undefined;
const failure = !result.success
? ffmpegFailure("silence", result, ffmpegProcessTimeout)
: undefined;
return {
success: result.success,
outputPath,
Expand Down Expand Up @@ -934,7 +937,7 @@ async function mixAudioTracks(
};
}
if (!result.success) {
const failure = ffmpegFailure("mix", result);
const failure = ffmpegFailure("mix", result, ffmpegProcessTimeout);
return {
success: false,
outputPath,
Expand Down Expand Up @@ -1000,20 +1003,21 @@ async function mixGroupMembers(
const outputDir = dirname(outputPath);
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });

const buildInputFilters = (ignoreKeyframes: boolean) =>
const buildInputFilters = (ignoreKeyframes: boolean, padMembers: boolean) =>
memberTracks.map((track, i) => {
const volumeFilter = buildVolumeExpression(track, ignoreKeyframes);
return buildTrackInputFilter(track, i, volumeFilter, totalDuration);
return buildTrackInputFilter(track, i, volumeFilter, totalDuration, padMembers);
});
const mixInputs = memberTracks.map((_, i) => `[a${i}]`).join("");

const runOnce = async (
useNormalize: boolean,
ignoreKeyframes = false,
): Promise<RunFfmpegResult> => {
const inputFilters = buildInputFilters(ignoreKeyframes);
const inputFilters = buildInputFilters(ignoreKeyframes, !useNormalize);
const padUnitySumOnceToFilm = `[mixed];[mixed]apad,asetpts=N/SR/TB,atrim=0:${formatFilterNumber(totalDuration)}`;
const mixFilter = useNormalize
? `${mixInputs}amix=inputs=${memberTracks.length}:duration=longest:dropout_transition=0:normalize=0[out]`
? `${mixInputs}amix=inputs=${memberTracks.length}:duration=longest:dropout_transition=0:normalize=0${padUnitySumOnceToFilm}[out]`
: // Every member is padded and trimmed to totalDuration above, and
// dropout_transition=0 keeps all N inputs active for that whole span.
// amix's default normalize therefore divides by exactly N throughout;
Expand Down Expand Up @@ -1089,7 +1093,7 @@ async function mixGroupMembers(
}

if (signal?.aborted || !result.success) {
const failure = ffmpegFailure("mix", result);
const failure = ffmpegFailure("mix", result, ffmpegProcessTimeout);
return { success: false, error: failure.detail, failure };
}
return { success: true, degradedAutomation };
Expand Down Expand Up @@ -1127,7 +1131,11 @@ async function mixThroughFloatSum(
timeout: config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,
});
if (!encoded.success) {
const failure = ffmpegFailure("mix", encoded);
const failure = ffmpegFailure(
"mix",
encoded,
config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,
);
return {
success: false,
outputPath,
Expand Down
Loading