From 2d756b51769c21f6111e2c4921387646138a4809 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:06:55 -0700 Subject: [PATCH 01/12] feat(cli): read a whole recording as voice, effect and music lines `hyperframes audio --read` turns an episode or reference video into time-stamped lines an agent can read or grep: sentences with rate, pitch and ending; pauses; sound effects grouped by identical sound, with band, attack, tail and level over the music; and the music bed split at its cut-outs into pieces that start, resume, return or are new. Effects and music come from stems separated by the service at HYPERFRAMES_AUDIO_STEMS_URL (vocals, drums, bass, guitar, piano, other). Without stems, effects are read from the mix outside speech; with neither stems nor a transcript, voice and effects are not read, and each case is reported in a warning. Results are cached per file content. --- docs/packages/cli.mdx | 42 +++ packages/cli/src/audio/analysis/decode.ts | 18 +- .../cli/src/audio/analysis/providers/api.ts | 31 +- .../src/audio/analysis/providers/shared.ts | 25 ++ packages/cli/src/audio/read/effects.test.ts | 120 ++++++ packages/cli/src/audio/read/effects.ts | 356 ++++++++++++++++++ packages/cli/src/audio/read/format.ts | 144 +++++++ packages/cli/src/audio/read/music.test.ts | 53 +++ packages/cli/src/audio/read/music.ts | 273 ++++++++++++++ packages/cli/src/audio/read/pitch.ts | 74 ++++ packages/cli/src/audio/read/read.test.ts | 190 ++++++++++ packages/cli/src/audio/read/read.ts | 205 ++++++++++ packages/cli/src/audio/read/signal.ts | 61 +++ packages/cli/src/audio/read/stems.ts | 148 ++++++++ packages/cli/src/audio/read/types.ts | 104 +++++ packages/cli/src/audio/read/voice.test.ts | 71 ++++ packages/cli/src/audio/read/voice.ts | 162 ++++++++ packages/cli/src/commands/audio.ts | 47 ++- 18 files changed, 2102 insertions(+), 22 deletions(-) create mode 100644 packages/cli/src/audio/read/effects.test.ts create mode 100644 packages/cli/src/audio/read/effects.ts create mode 100644 packages/cli/src/audio/read/format.ts create mode 100644 packages/cli/src/audio/read/music.test.ts create mode 100644 packages/cli/src/audio/read/music.ts create mode 100644 packages/cli/src/audio/read/pitch.ts create mode 100644 packages/cli/src/audio/read/read.test.ts create mode 100644 packages/cli/src/audio/read/read.ts create mode 100644 packages/cli/src/audio/read/signal.ts create mode 100644 packages/cli/src/audio/read/stems.ts create mode 100644 packages/cli/src/audio/read/types.ts create mode 100644 packages/cli/src/audio/read/voice.test.ts create mode 100644 packages/cli/src/audio/read/voice.ts diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 362b420d520..adf91ad3f8b 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -583,6 +583,48 @@ first, and the result warns when the cut part held the file's loudest moment. Word timing is not here: use [`transcribe`](#transcribe). +#### Reading a whole recording + +`--read` turns one recording — an episode, a reference video — into +time-stamped lines an agent can read or grep, instead of measurements of one sound: + +```bash +npx hyperframes audio refs/episode.mp4 --read # the whole recording +npx hyperframes audio refs/episode.mp4 --read --range 60,70 # only what overlaps 60–70 s +``` + +- **voice**: one line per sentence with its text, speaking rate against the + recording's mean, pitch when it is 3 semitones or more off the median, and a + rising or falling end; pauses of 0.4 s or more between sentences. +- **effects**: each sound effect with a kind (`riser`, `boom`, `thump`, `whoosh`, + `click`, `tick`, `ding`, `hit`), the group of identical sounds it belongs to + (`S1` is the most used), band, attack, tail, how far it stands over the music, + and where it falls in a pause. Quick runs of one sound are a single line with a + count; hits on the music's beat are the music's own drums, and sounds more than + 20 dB under the recording are separation residue nobody hears; both are left out. +- **music**: the bed split at its cut-outs, each part naming its piece, the + instruments that carry it and its tempo, whether it starts, resumes after a + break, returns or is new, and where it fades. + +Sentences come from a transcript: `--transcript `, or +`.transcript.json` beside the recording. Without one, voice lines +carry no words. + +Effects and music are read from stems. At `--quality best` (the default) the +recording is separated into vocals, drums, bass, guitar, piano and other by the +service at `HYPERFRAMES_AUDIO_STEMS_URL`, with `HYPERFRAMES_AUDIO_API_KEY`: it +is uploaded as 192 kb/s MP3 through the same origin's direct-upload endpoints, +`POST` starts a job, and `GET /` is polled until the stems are ready +(`HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS`, 30 minutes by default). Effects are read +from the drums stem and the music from the other four. Without stems, effects are +read from the mix outside speech and the music is not read; with neither stems nor +a transcript, voice and effects are not read either, because the mix alone does +not tell speech from music. Each case says so in a warning. + +The result is computed once per file content and kept in `.hyperframes/read-audio` +under the project, with the stems; the text output ends with the path of the +whole recording's text, and `--json` returns the analysis. + Every measurement comes from a provider ladder, tried in this order until one answers: diff --git a/packages/cli/src/audio/analysis/decode.ts b/packages/cli/src/audio/analysis/decode.ts index ef86377d24c..00fe4a4056c 100644 --- a/packages/cli/src/audio/analysis/decode.ts +++ b/packages/cli/src/audio/analysis/decode.ts @@ -55,6 +55,12 @@ export interface DecodeWindow { seconds?: number; } +/** Decode to this rate, and to one channel, instead of the file's own layout. */ +export interface DecodeFormat { + sampleRate?: number; + mono?: boolean; +} + export interface Decoded { signal: AudioSignal; stream: AudioStreamInfo; @@ -65,9 +71,15 @@ export async function decodeAudio( file: string, window: DecodeWindow = {}, probed?: AudioStreamInfo, + format: DecodeFormat = {}, ): Promise { - const info = probed ?? (await probeAudio(file)); - if (!info) return null; + const probedInfo = probed ?? (await probeAudio(file)); + if (!probedInfo) return null; + const info = { + ...probedInfo, + sampleRate: format.sampleRate ?? probedInfo.sampleRate, + channels: format.mono ? 1 : probedInfo.channels, + }; const seek = window.from ? ["-ss", String(window.from)] : []; const length = window.seconds === undefined ? [] : ["-t", String(window.seconds)]; const { stdout } = await run( @@ -81,6 +93,8 @@ export async function decodeAudio( ...length, "-map", "0:a:0", + ...(format.mono ? ["-ac", "1"] : []), + ...(format.sampleRate ? ["-ar", String(format.sampleRate)] : []), "-f", "f32le", "-acodec", diff --git a/packages/cli/src/audio/analysis/providers/api.ts b/packages/cli/src/audio/analysis/providers/api.ts index 29451eccc87..ddf1e65a7b0 100644 --- a/packages/cli/src/audio/analysis/providers/api.ts +++ b/packages/cli/src/audio/analysis/providers/api.ts @@ -10,7 +10,15 @@ import { type AudioSignal, type Snapper, } from "@hyperframes/core/audio-analysis"; -import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared.js"; +import { + AVAILABLE, + encodeWav, + perSignal, + rounded, + serviceConfig, + serviceFailure, + unavailable, +} from "./shared.js"; /** * The audio analysis service contract. @@ -115,12 +123,8 @@ function uploaded(signal: AudioSignal): AudioSignal { const askedTogether = (capability: AudioApiCapability): AudioApiCapability[] => capability === "beats" || capability === "downbeats" ? ["beats", "downbeats"] : [capability]; -function config() { - const url = process.env[URL_ENV]?.trim(); - const key = process.env[KEY_ENV]?.trim(); - const timeout = Number(process.env[TIMEOUT_ENV]); - return { url, key, timeoutMs: timeout > 0 ? timeout : DEFAULT_TIMEOUT_MS }; -} +const config = () => + serviceConfig({ url: URL_ENV, key: KEY_ENV, timeout: TIMEOUT_ENV }, DEFAULT_TIMEOUT_MS); const isNumberList = (value: unknown): value is number[] => Array.isArray(value) && value.every((t) => typeof t === "number" && Number.isFinite(t)); @@ -146,17 +150,6 @@ function checked(capability: AudioApiCapability, body: AudioApiResponse): void { throw new Error(`the service answered ${capability} in a shape this CLI cannot read`); } -async function failure(response: Response): Promise { - const text = await response.text().catch(() => ""); - let detail = text.slice(0, 200); - try { - detail = (JSON.parse(text) as { error?: { message?: string } }).error?.message ?? detail; - } catch { - // not JSON: keep the text - } - return new Error(`the service answered ${response.status}${detail ? `: ${detail}` : ""}`); -} - const uploadedAsset = perSignal(async (analyzed: AudioSignal) => { const { url, key, timeoutMs } = config(); if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`); @@ -202,7 +195,7 @@ async function ask(analyzed: AudioSignal, capabilities: AudioApiCapability[]) { body: JSON.stringify(request), signal: AbortSignal.timeout(timeoutMs), }); - if (!response.ok) throw await failure(response); + if (!response.ok) throw await serviceFailure(response, "the service"); const body = (await response.json()) as { data?: AudioApiResponse } | null; return (body?.data ?? body ?? {}) as AudioApiResponse; } diff --git a/packages/cli/src/audio/analysis/providers/shared.ts b/packages/cli/src/audio/analysis/providers/shared.ts index c6e88153d98..689a9e3b400 100644 --- a/packages/cli/src/audio/analysis/providers/shared.ts +++ b/packages/cli/src/audio/analysis/providers/shared.ts @@ -74,3 +74,28 @@ export function downbeatsWithMeter(beats: number[], downbeats: number[]): Downbe const round = (t: number) => Math.round(t * 1000) / 1000; export const rounded = (times: readonly number[]) => times.map(round); + +/** A remote service's URL, key and timeout, read from the environment each call. */ +export function serviceConfig( + env: { url: string; key: string; timeout: string }, + defaultMs: number, +) { + const timeout = Number(process.env[env.timeout]); + return { + url: process.env[env.url]?.trim(), + key: process.env[env.key]?.trim(), + timeoutMs: timeout > 0 ? timeout : defaultMs, + }; +} + +/** A failed response as an error carrying the service's own message when it sent one. */ +export async function serviceFailure(response: Response, service: string): Promise { + const text = await response.text().catch(() => ""); + let detail = text.slice(0, 200); + try { + detail = (JSON.parse(text) as { error?: { message?: string } }).error?.message ?? detail; + } catch { + // not JSON: keep the text + } + return new Error(`${service} answered ${response.status}${detail ? `: ${detail}` : ""}`); +} diff --git a/packages/cli/src/audio/read/effects.test.ts b/packages/cli/src/audio/read/effects.test.ts new file mode 100644 index 00000000000..2d6005b841e --- /dev/null +++ b/packages/cli/src/audio/read/effects.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; +import { findEffects } from "./effects.js"; +import { READ_RATE } from "./signal.js"; + +let seed = 11; +const noise = () => + ((seed = (seed * 1_103_515_245 + 12_345) % 2_147_483_648) / 2_147_483_648) * 2 - 1; + +function track(seconds: number, ...sounds: ((t: number) => number)[]): Float32Array { + seed = 11; + const out = new Float32Array(Math.round(seconds * READ_RATE)); + for (let i = 0; i < out.length; i++) { + const t = i / READ_RATE; + out[i] = 0.0005 * noise() + sounds.reduce((sum, sound) => sum + sound(t), 0); + } + return out; +} + +const boom = (at: number) => (t: number) => + t < at ? 0 : 0.8 * Math.exp(-(t - at) / 0.35) * Math.sin(2 * Math.PI * 55 * (t - at)); +const click = (at: number) => (t: number) => + t < at || t > at + 0.03 ? 0 : 0.6 * Math.exp(-(t - at) / 0.006) * noise(); +const riser = (at: number, length: number) => (t: number) => + t < at || t > at + length + 0.05 + ? 0 + : 0.5 * + Math.min(1, (t - at) / length) ** 2 * + noise() * + (t > at + length ? 1 - (t - at - length) / 0.05 : 1); + +describe("findEffects", () => { + it("names a low boom, a short bright tick and a long riser, each once", () => { + const source = track(8, boom(1), click(3.2), riser(5, 1.2)); + const { events } = findEffects({ + source, + sampleRate: READ_RATE, + music: null, + heard: null, + beats: [], + speech: [], + }); + expect(events.map((e) => [Math.round(e.t * 10) / 10, e.kind])).toEqual([ + [1, "boom"], + [3.2, "tick"], + [5, "riser"], + ]); + expect(events[0]).toMatchObject({ band: "low", count: 1, overMusicDb: null }); + expect(events[0]!.tail).toBeGreaterThan(0.6); + expect(events[2]!.attackMs).toBeGreaterThan(600); + }); + + it("groups the same sound and reports a quick run of it as one train", () => { + const source = track(6, click(1), click(1.12), click(1.24), click(4)); + const { events, groups } = findEffects({ + source, + sampleRate: READ_RATE, + music: null, + heard: null, + beats: [], + speech: [], + }); + expect(groups).toEqual([{ id: "S1", kind: "tick", count: 4 }]); + expect(events.map((e) => [e.t, e.count])).toEqual([ + [1, 3], + [4, 1], + ]); + }); + + it("leaves out a sound that keeps landing on the music's beat, and skips onsets inside speech", () => { + const beats = Array.from({ length: 12 }, (_, i) => 0.5 + i * 0.5); + const source = track(7, ...beats.map((b) => click(b)), boom(6.2)); + const result = findEffects({ + source, + sampleRate: READ_RATE, + music: null, + heard: null, + beats, + speech: [], + }); + expect(result.drumHits).toBe(12); + expect(result.events.map((e) => e.kind)).toEqual(["boom"]); + const muted = findEffects({ + source: track(4, boom(1)), + sampleRate: READ_RATE, + music: null, + heard: null, + beats: [], + speech: [{ start: 0.8, end: 1.5 }], + }); + expect(muted.events).toEqual([]); + }); + + it("measures an effect against the music playing under it", () => { + const source = track(4, boom(2)); + const music = track(4, (t) => 0.05 * Math.sin(2 * Math.PI * 440 * t)); + const [event] = findEffects({ + source, + sampleRate: READ_RATE, + music, + heard: null, + beats: [], + speech: [], + }).events; + expect(event!.overMusicDb).toBeGreaterThan(10); + }); + + it("leaves out stem residue far under the recording, and gives no level over music that is out", () => { + const quiet = track(4, (t) => 0.01 * boom(1)(t), boom(3)); + const loud = track(4, (t) => (t < 2 ? 0.5 * Math.sin(2 * Math.PI * 220 * t) : 0)); + const { events } = findEffects({ + source: quiet, + sampleRate: READ_RATE, + music: track(4, () => 0), + heard: quiet.map((v, i) => v + loud[i]!), + beats: [], + speech: [], + }); + expect(events.map((e) => [Math.round(e.t), e.overMusicDb])).toEqual([[3, null]]); + }); +}); diff --git a/packages/cli/src/audio/read/effects.ts b/packages/cli/src/audio/read/effects.ts new file mode 100644 index 00000000000..db2a1d85d53 --- /dev/null +++ b/packages/cli/src/audio/read/effects.ts @@ -0,0 +1,356 @@ +import { bandFramesOf, type Span } from "@hyperframes/core/audio-analysis"; +import { asSignal, levels, quantile, round } from "./signal.js"; +import type { Band, Effect, EffectGroup, EffectKind, Effects } from "./types.js"; + +const HOP = 0.01; +const FLOOR_FRAMES = 150; +const FLOOR_EVERY = 10; +const RISE_DB = 20; +const STEP_DB = 6; +const AUDIBLE_DB = -60; +const MASKED_DB = 20; +const MUSIC_OUT_DB = -60; +const TAIL_DB = 30; +const LONGEST = 4; +const SAME_SOUND = 0.95; +const TRAIN_GAP = 0.2; +const BEAT_SLACK = 0.05; +const ON_BEAT_SHARE = 0.7; +const DRUM_GROUP_MIN = 8; +const LOW_HZ = 150; +const BAND_FPS = 100; + +interface Found { + t: number; + peak: number; + end: number; + peakDb: number; + riseDb: number; +} + +/** A rolling low percentile of the level: what was playing before each frame. */ +function floors(db: Float32Array): Float32Array { + const out = new Float32Array(db.length); + let current = db[0] ?? -120; + for (let f = 0; f < db.length; f++) { + if (f % FLOOR_EVERY === 0 && f > 0) + current = quantile(db.subarray(Math.max(0, f - FLOOR_FRAMES), f), 0.2); + out[f] = current; + } + return out; +} + +const BLOCK = 0.001; + +/** The first millisecond that rises halfway out of the floor: frame levels alone place an onset up to a hop early. */ +function refined( + samples: Float32Array, + sampleRate: number, + frame: number, + floorDb: number, +): number { + const block = Math.max(1, Math.round(BLOCK * sampleRate)); + const from = Math.max(0, Math.round((frame - 2) * HOP * sampleRate)); + const to = Math.min(samples.length, Math.round((frame + 2) * HOP * sampleRate)); + for (let a = from; a + block <= to; a += block) { + let power = 0; + for (let i = a; i < a + block; i++) power += samples[i]! ** 2; + if (10 * Math.log10(power / block + 1e-12) >= floorDb + RISE_DB / 2) return a / sampleRate; + } + return frame * HOP; +} + +/** How far a sound found at frame `f` reaches: back to where it left the floor, on to its peak and through its tail. */ +function extent(db: Float32Array, f: number, floor: number) { + let onset = f; + while (onset > Math.max(0, f - 50) && db[onset - 1]! > floor + STEP_DB) onset--; + let peak = f; + let end = f; + const last = Math.min(db.length - 1, f + LONGEST / HOP); + for (let g = f; g <= last; g++) { + if (db[g]! > db[peak]!) peak = g; + end = g; + if (g > peak && (db[g]! < db[peak]! - TAIL_DB || db[g]! < floor + 3)) break; + } + return { onset, peak, end }; +} + +function startsHere(db: Float32Array, f: number, floor: number): boolean { + const level = db[f]!; + if (level < AUDIBLE_DB || level - floor < RISE_DB) return false; + return level - Math.min(db[f - 1]!, db[f - 2]!, db[f - 3]!, db[f - 4]!, db[f - 5]!) >= STEP_DB; +} + +function detect( + db: Float32Array, + samples: Float32Array, + sampleRate: number, + skip: (t: number) => boolean, +): Found[] { + const floor = floors(db); + const found: Found[] = []; + for (let f = 5; f < db.length; f++) { + if (!startsHere(db, f, floor[f]!)) continue; + const { onset, peak, end } = extent(db, f, floor[f]!); + const t = refined(samples, sampleRate, onset, floor[f]!); + if (!skip(t)) + found.push({ + t, + peak: peak * HOP, + end: end * HOP, + peakDb: db[peak]!, + riseDb: db[peak]! - floor[f]!, + }); + f = Math.max(end, onset + 8); + } + return found; +} + +const BIN_HZ = 22050 / 1024; +const EDGES = (() => { + const edges: number[] = []; + for (let hz = 30; hz < 9000; hz *= 2 ** (1 / 5)) edges.push(Math.max(1, Math.round(hz / BIN_HZ))); + return edges; +})(); +const bandHz = (k: number) => 30 * 2 ** ((k + 0.5) / 5); +/** Bins in each band: core's band frames hold the mean power per bin, so a band's energy is that times its width. */ +const WIDTH = EDGES.slice(0, -1).map((edge, k) => Math.max(edge + 1, EDGES[k + 1]!) - edge); +const energies = (power: Float64Array) => Array.from(power, (v, k) => Math.expm1(v) * WIDTH[k]!); +const BANDS_PER_PRINT = 5; + +function shape(bands: Float64Array[], from: number, to: number): Float64Array { + const a = Math.max(0, Math.floor(from * BAND_FPS)); + const b = Math.max(a + 1, Math.min(bands.length, Math.ceil(to * BAND_FPS))); + const size = bands[0]?.length ?? 0; + const mean = new Float64Array(size); + for (let f = a; f < b; f++) for (let k = 0; k < size; k++) mean[k]! += bands[f]![k]! / (b - a); + return mean; +} + +function lowShare(power: Float64Array): number { + const e = energies(power); + const total = e.reduce((a, b) => a + b, 0); + const low = e.reduce((sum, p, k) => sum + (bandHz(k) < LOW_HZ ? p : 0), 0); + return total > 0 ? low / total : 0; +} + +function centroidHz(power: Float64Array): number { + const e = energies(power); + const weight = e.reduce((a, b) => a + b, 0); + const sum = e.reduce((acc, p, k) => acc + p * Math.log2(bandHz(k)), 0); + return weight > 0 ? 2 ** (sum / weight) : 0; +} + +const tonal = (power: Float64Array) => { + const sorted = Float64Array.from(power).sort(); + return sorted[sorted.length - 1]! - sorted[Math.floor(sorted.length * 0.75)]! >= 2.5; +}; + +interface Shape { + length: number; + attack: number; + tail: number; + band: Band; + centroid: number; + tonal: boolean; +} + +/** Tried in order; a sound none of them fits is a hit. */ +const KINDS: [(s: Shape) => boolean, (s: Shape) => EffectKind][] = [ + [(s) => s.length >= 0.8 && s.attack >= 0.45 * s.length, () => "riser"], + [(s) => s.band === "low", (s) => (s.tail >= 0.6 ? "boom" : "thump")], + [(s) => s.length <= 0.15, (s) => (s.centroid >= 3000 ? "tick" : "click")], + [(s) => s.tonal && s.tail >= 0.4, () => "ding"], + [(s) => s.attack >= 0.06 && s.length <= 1.5, () => "whoosh"], +]; + +function bandOf(low: number, centroid: number): Band { + if (low >= 0.5) return "low"; + return centroid >= 2500 ? "high" : "mid"; +} + +function kindOf(found: Found, power: Float64Array): { kind: EffectKind; band: Band } { + const centroid = centroidHz(power); + const shape: Shape = { + length: found.end - found.t, + attack: found.peak - found.t, + tail: found.end - found.peak, + band: bandOf(lowShare(power), centroid), + centroid, + tonal: tonal(power), + }; + const kind = KINDS.find(([fits]) => fits(shape))?.[1](shape) ?? "hit"; + return { kind, band: shape.band }; +} + +/** The sound's spectral shape: the square root of each octave's share of its energy, so loudness does not count. */ +function printOf(power: Float64Array): Float64Array { + const e = energies(power); + const octaves = new Float64Array(Math.ceil(e.length / BANDS_PER_PRINT)); + e.forEach((p, k) => (octaves[Math.floor(k / BANDS_PER_PRINT)]! += p)); + const total = octaves.reduce((a, b) => a + b, 0); + return octaves.map((p) => (total > 0 ? Math.sqrt(p / total) : 0)); +} + +function cosine(a: Float64Array, b: Float64Array): number { + let dot = 0; + let na = 0; + let nb = 0; + for (let k = 0; k < a.length; k++) { + dot += a[k]! * b[k]!; + na += a[k]! ** 2; + nb += b[k]! ** 2; + } + return na && nb ? dot / Math.sqrt(na * nb) : 0; +} + +interface Measured extends Found { + kind: EffectKind; + band: Band; + print: Float64Array; + group: number; +} + +function grouped(events: Measured[]): Float64Array[] { + const prints: { sum: Float64Array; n: number; length: number }[] = []; + for (const event of events) { + const length = event.end - event.t; + let best = -1; + let bestScore = SAME_SOUND; + prints.forEach((p, i) => { + const ratio = length / p.length; + if (ratio < 0.5 || ratio > 2) return; + const score = cosine( + event.print, + p.sum.map((v) => v / p.n), + ); + if (score >= bestScore) { + best = i; + bestScore = score; + } + }); + if (best < 0) { + prints.push({ sum: Float64Array.from(event.print), n: 1, length }); + best = prints.length - 1; + } else { + const p = prints[best]!; + event.print.forEach((v, k) => (p.sum[k]! += v)); + p.length = (p.length * p.n + length) / (p.n + 1); + p.n++; + } + event.group = best; + } + return prints.map((p) => p.sum); +} + +function onBeat(t: number, beats: readonly number[]): boolean { + let lo = 0; + let hi = beats.length; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (beats[mid]! < t) lo = mid + 1; + else hi = mid; + } + const near = [beats[lo - 1], beats[lo]].filter((b): b is number => b !== undefined); + return near.some((b) => Math.abs(b - t) <= BEAT_SLACK); +} + +function majority(kinds: EffectKind[]): EffectKind { + const counts = new Map(); + for (const k of kinds) counts.set(k, (counts.get(k) ?? 0) + 1); + return [...counts].sort((a, b) => b[1] - a[1])[0]![0]; +} + +export interface EffectInput { + /** The track effects are found in: the drum stem, or the whole mix without stems. */ + source: Float32Array; + sampleRate: number; + /** The music without drums, to say how far an effect stands over it. */ + music: Float32Array | null; + /** The whole recording, when the source is a stem: an effect far under it is separation residue nobody hears. */ + heard: Float32Array | null; + beats: readonly number[]; + /** Spans where an onset is speech, not an effect (the mix only). */ + speech: readonly Span[]; +} + +function measured(input: EffectInput): Measured[] { + const db = levels(input.source, input.sampleRate, HOP); + const inSpeech = (t: number) => input.speech.some((s) => t >= s.start && t < s.end); + const heardDb = input.heard ? levels(input.heard, input.sampleRate, HOP, 0.5) : null; + const masked = (f: Found) => { + const level = heardDb?.[Math.round(f.peak / HOP)]; + return level !== undefined && f.peakDb < level - MASKED_DB; + }; + const found = detect(db, input.source, input.sampleRate, inSpeech).filter((f) => !masked(f)); + const bands = found.length ? bandFramesOf(asSignal(input.source, input.sampleRate)).bands : []; + const out: Measured[] = found.map((f) => { + const power = shape(bands, f.t, Math.min(f.end, f.t + 0.4)); + return { ...f, ...kindOf(f, power), print: printOf(power), group: -1 }; + }); + grouped(out); + return out; +} + +/** Groups that keep landing on the music's beat are its own drums, not effects. */ +function drumGroups(members: Map, beats: readonly number[]): Set { + const drums = new Set(); + if (!beats.length) return drums; + for (const [id, list] of members) { + const onBeats = list.filter((m) => onBeat(m.t, beats)).length; + if (list.length >= DRUM_GROUP_MIN && onBeats >= ON_BEAT_SHARE * list.length) drums.add(id); + } + return drums; +} + +function effectOf( + m: Measured, + group: string, + kind: EffectKind, + musicDb: Float32Array | null, +): Effect { + const musicLevel = musicDb?.[Math.round(m.peak / HOP)]; + const musicPlays = musicLevel !== undefined && musicLevel > MUSIC_OUT_DB; + return { + t: round(m.t), + end: round(m.end), + kind, + group, + band: m.band, + attackMs: Math.round((m.peak - m.t) * 1000), + tail: round(m.end - m.peak), + riseDb: Math.round(m.riseDb), + overMusicDb: musicPlays ? Math.round(m.peakDb - musicLevel) : null, + count: 1, + }; +} + +export function findEffects(input: EffectInput): Effects { + const sounds = measured(input); + const members = new Map(); + for (const m of sounds) members.set(m.group, [...(members.get(m.group) ?? []), m]); + const drums = drumGroups(members, input.beats); + const kept = [...members] + .filter(([id]) => !drums.has(id)) + .sort((a, b) => b[1].length - a[1].length); + const names = new Map(kept.map(([id], i) => [id, `S${i + 1}`])); + const groups: EffectGroup[] = kept.map(([id, list]) => ({ + id: names.get(id)!, + kind: majority(list.map((m) => m.kind)), + count: list.length, + })); + const kindOfGroup = new Map(groups.map((g) => [g.id, g.kind])); + const musicDb = input.music ? levels(input.music, input.sampleRate, HOP, 0.5) : null; + const events: Effect[] = []; + for (const m of sounds) { + if (drums.has(m.group)) continue; + const group = names.get(m.group)!; + const previous = events[events.length - 1]; + if (previous?.group === group && m.t - previous.end <= TRAIN_GAP) { + previous.end = round(m.end); + previous.count++; + } else events.push(effectOf(m, group, kindOfGroup.get(group) ?? m.kind, musicDb)); + } + const drumHits = [...drums].reduce((n, id) => n + members.get(id)!.length, 0); + return { events, groups, drumHits }; +} diff --git a/packages/cli/src/audio/read/format.ts b/packages/cli/src/audio/read/format.ts new file mode 100644 index 00000000000..bf54b06d0d6 --- /dev/null +++ b/packages/cli/src/audio/read/format.ts @@ -0,0 +1,144 @@ +import type { Span } from "@hyperframes/core/audio-analysis"; +import { semitones } from "./pitch.js"; +import type { Effect, Entry, MusicSegment, ReadAudio, Sentence, Voice } from "./types.js"; + +const PITCH_NOTE_SEMITONES = 3; +const KIND_WIDTH = 7; + +interface Line { + start: number; + end: number | null; + kind: string; + text: string; +} + +const s = (t: number) => t.toFixed(1); +const signed = (db: number) => `${db >= 0 ? "+" : ""}${db} dB`; + +function when(line: Line): string { + return line.end === null ? s(line.start) : `${s(line.start)}–${s(line.end)}`; +} + +function voiceLine(sentence: Sentence, voice: Voice): Line { + const { unit, meanRate, medianPitchHz } = voice; + const parts = [sentence.text === null ? "(no transcript)" : `"${sentence.text}"`]; + if (sentence.rate !== null) + parts.push(`${sentence.rate} ${unit}/s${meanRate !== null ? ` (mean ${meanRate})` : ""}`); + if ( + sentence.pitchHz !== null && + medianPitchHz !== null && + Math.abs(semitones(sentence.pitchHz, medianPitchHz)) >= PITCH_NOTE_SEMITONES + ) + parts.push(`pitch ${sentence.pitchHz} Hz (median ${medianPitchHz})`); + if (sentence.ending && sentence.ending !== "level") parts.push(`${sentence.ending} end`); + return { start: sentence.start, end: sentence.end, kind: "voice", text: parts.join(" ") }; +} + +function placeOf(t: number, pauses: readonly Span[]): string | null { + const pause = pauses.find((p) => t >= p.start && t < p.end); + return pause ? `${(t - pause.start).toFixed(1)} s into the pause` : null; +} + +function standing(effect: Effect, musicRead: boolean): string { + if (effect.overMusicDb !== null) return `${signed(effect.overMusicDb)} over music`; + return musicRead ? "while the music is out" : `${signed(effect.riseDb)} over what played before`; +} + +function effectLine(effect: Effect, pauses: readonly Span[], musicRead: boolean): Line { + const parts = [ + `${effect.kind}${effect.count > 1 ? ` ×${effect.count}` : ""} (${effect.group}): ${effect.band}`, + `attack ${effect.attackMs} ms`, + `tail ${effect.tail.toFixed(1)} s`, + standing(effect, musicRead), + ]; + const place = placeOf(effect.t, pauses); + if (place) parts.push(place); + return { start: effect.t, end: null, kind: "effect", text: parts.join(", ") }; +} + +const ENTRY: Record = { + start: "", + new: ", first appearance after a break", + resumes: ", resumes after a break", + returns: ", returns", +}; + +function musicLine(segment: MusicSegment, analysis: ReadAudio): Line { + const piece = analysis.music!.pieces.find((p) => p.id === segment.piece)!; + const about = [piece.instruments.join(", "), segment.bpm ? `${segment.bpm} BPM` : null] + .filter(Boolean) + .join(" · "); + const fade = segment.fadeFrom !== null ? `, fades from ${s(segment.fadeFrom)}` : ""; + return { + start: segment.start, + end: segment.end, + kind: "music", + text: `piece ${piece.id}${about ? ` (${about})` : ""}${ENTRY[segment.entry]}${fade}`, + }; +} + +function overlaps(line: Line, range?: [number, number]): boolean { + if (!range) return true; + return line.start < range[1] && (line.end ?? line.start) >= range[0]; +} + +function header(analysis: ReadAudio): string[] { + const { voice, effects, music } = analysis; + const separated = + analysis.separation === "stems" ? `stems: ${analysis.stems.join(", ")}` : "not separated"; + const lines = [`${analysis.file} · ${s(analysis.duration)} s · ${separated}`]; + if (voice) { + const rate = voice.meanRate !== null ? `mean ${voice.meanRate} ${voice.unit}/s` : null; + const pitch = voice.medianPitchHz !== null ? `median pitch ${voice.medianPitchHz} Hz` : null; + lines.push( + `voice: ${voice.sentences.length} lines, ${voice.pauses.length} pauses${[rate, pitch] + .filter(Boolean) + .map((p) => `, ${p}`) + .join("")}`, + ); + } + if (effects) { + const top = effects.groups + .slice(0, 6) + .map((g) => `${g.id} ${g.kind} ×${g.count}`) + .join(", "); + lines.push( + `effects: ${effects.events.length} in ${effects.groups.length} sounds${top ? ` (${top})` : ""}${ + effects.drumHits ? `; ${effects.drumHits} drum hits on the beat left out` : "" + }`, + ); + } + if (music) lines.push(`music: ${music.pieces.length} pieces, ${music.breaks.length} breaks`); + return lines; +} + +/** The recording as time-stamped lines, all of it or the events overlapping `range`. */ +export function formatRead(analysis: ReadAudio, range?: [number, number]): string { + const { voice, effects } = analysis; + const pauses = voice?.pauses ?? []; + const lines: Line[] = [ + ...(voice?.sentences.map((x) => voiceLine(x, voice)) ?? []), + ...pauses.map((p) => ({ + start: p.start, + end: p.end, + kind: "pause", + text: `${(p.end - p.start).toFixed(1)} s`, + })), + ...(effects?.events.map((e) => effectLine(e, pauses, analysis.music !== null)) ?? []), + ...(analysis.music?.segments.map((m) => musicLine(m, analysis)) ?? []), + ...(analysis.music?.breaks.map((b) => ({ + start: b.start, + end: b.end, + kind: "music", + text: `break ${(b.end - b.start).toFixed(1)} s`, + })) ?? []), + ] + .filter((line) => overlaps(line, range)) + .sort((a, b) => a.start - b.start || (a.end ?? a.start) - (b.end ?? b.start)); + const width = Math.max(0, ...lines.map((l) => when(l).length)); + const body = lines.map((l) => `${when(l).padEnd(width)} ${l.kind.padEnd(KIND_WIDTH)} ${l.text}`); + const warnings = analysis.warnings.map((w) => `! ${w}`); + return [...header(analysis), "", ...body, ...(warnings.length ? ["", ...warnings] : [])].join( + "\n", + ); +} diff --git a/packages/cli/src/audio/read/music.test.ts b/packages/cli/src/audio/read/music.test.ts new file mode 100644 index 00000000000..2a8f83c4672 --- /dev/null +++ b/packages/cli/src/audio/read/music.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { readMusic } from "./music.js"; +import { READ_RATE } from "./signal.js"; + +const chord = (hz: number[]) => (t: number) => + hz.reduce((sum, f) => sum + 0.1 * Math.sin(2 * Math.PI * f * t), 0); +const C_MAJOR = chord([261.63, 329.63, 392]); +const F_SHARP_MINOR = chord([369.99, 440, 554.37]); + +function piano(seconds: number, parts: [number, number, (t: number) => number][]): Float32Array { + const out = new Float32Array(Math.round(seconds * READ_RATE)); + for (const [from, to, sound] of parts) + for (let i = Math.round(from * READ_RATE); i < Math.round(to * READ_RATE); i++) + out[i] = sound(i / READ_RATE); + return out; +} + +describe("readMusic", () => { + it("splits the bed at its cut-outs and tells a new piece from one that returns", () => { + const stem = piano(26, [ + [0, 8, C_MAJOR], + [9, 16, F_SHARP_MINOR], + [17, 24, C_MAJOR], + ]); + const { music } = readMusic({ stems: { piano: stem }, drums: null, sampleRate: READ_RATE }); + expect( + music.segments.map((s) => [Math.round(s.start), Math.round(s.end), s.piece, s.entry]), + ).toEqual([ + [0, 8, 1, "start"], + [9, 16, 2, "new"], + [17, 24, 1, "returns"], + ]); + expect(music.breaks.map((b) => [Math.round(b.start), Math.round(b.end)])).toEqual([ + [8, 9], + [16, 17], + ]); + expect(music.pieces[0]).toMatchObject({ id: 1, instruments: ["piano"], firstAt: 0 }); + expect(music.energy.map((e) => e.start)).toEqual([0]); + }); + + it("names the stems that carry a piece, loudest first, and notes a fade at its end", () => { + const fadeOut = (t: number) => C_MAJOR(t) * Math.min(1, Math.max(0.001, (10 - t) / 3)); + const loud = piano(10, [[0, 10, fadeOut]]); + const quiet = loud.map((v) => v * 0.6); + const { music } = readMusic({ + stems: { piano: loud, guitar: quiet, bass: new Float32Array(loud.length) }, + drums: null, + sampleRate: READ_RATE, + }); + expect(music.pieces[0]!.instruments).toEqual(["piano", "guitar"]); + expect(music.segments[0]!.fadeFrom).toBeGreaterThan(6.5); + }); +}); diff --git a/packages/cli/src/audio/read/music.ts b/packages/cli/src/audio/read/music.ts new file mode 100644 index 00000000000..c5fb5aebf28 --- /dev/null +++ b/packages/cli/src/audio/read/music.ts @@ -0,0 +1,273 @@ +import { bandFramesOf, medianInterval, rhythm } from "@hyperframes/core/audio-analysis"; +import { asSignal, levels, median, mixed, round } from "./signal.js"; +import type { Entry, Music, MusicSegment, Piece } from "./types.js"; + +const HOP = 0.1; +const ABSENT_DB = -55; +const CUT_DB = 25; +const RUNNING_FRAMES = 200; +const LEVEL_WINDOW = 0.4; +const CONTEXT = 2; +const SHORTEST_BREAK = 0.5; +const LONGEST_REST = 1; +const SHORTEST_SEGMENT = 1; +const SAME_PIECE = 0.92; +const SAME_TEMPO = 0.06; +const SAME_MIX = 0.25; +const RESUMING_GAP = 5; +const RESUMING_PIECE = 0.85; +const RESUMING_MIX = 0.35; +const FADE_DB = 10; +const FADE_WINDOW = 3; +const ENERGY_WINDOW = 30; +const LEADING_SHARE = 0.2; + +export interface MusicInput { + /** Every pitched stem, by name: bass, guitar, piano, other. */ + stems: Record; + drums: Float32Array | null; + sampleRate: number; +} + +interface Run { + start: number; + end: number; +} + +type Span = { quiet: boolean; start: number; end: number }; + +function joined(spans: Span[], tooShort: (span: Span) => boolean): Span[] { + const out: Span[] = []; + for (const span of spans) { + const last = out[out.length - 1]; + if (last && (tooShort(span) || last.quiet === span.quiet)) last.end = span.end; + else out.push({ ...span }); + } + return out; +} + +function powerDb(db: Float32Array, start: number, end: number): number { + const a = Math.max(0, Math.round(start / HOP)); + const b = Math.min(db.length, Math.round(end / HOP)); + let sum = 0; + for (let f = a; f < b; f++) sum += 10 ** (db[f]! / 10); + return b > a ? 10 * Math.log10(sum / (b - a) || 1e-12) : Number.NEGATIVE_INFINITY; +} + +/** A break holds the music far under what plays around it, or out altogether, across its whole length. */ +function holdsUp(db: Float32Array, span: Span): boolean { + const inside = powerDb(db, span.start, span.end); + const around = Math.max( + powerDb(db, span.start - CONTEXT, span.start), + powerDb(db, span.end, span.end + CONTEXT), + ); + return inside < ABSENT_DB || inside < around - CUT_DB; +} + +const tooShort = (span: Span) => + span.end - span.start < (span.quiet ? SHORTEST_BREAK : SHORTEST_SEGMENT); + +function candidates( + quiet: Uint8Array, + from: number, + to: number, + absorbed: (span: Span) => boolean = tooShort, +): Span[] { + const spans: Span[] = []; + for (let f = from; f < to; f++) { + const isQuiet = quiet[f] === 1; + const last = spans[spans.length - 1]; + if (last && last.quiet === isQuiet) last.end = (f + 1) * HOP; + else spans.push({ quiet: isQuiet, start: f * HOP, end: (f + 1) * HOP }); + } + return joined(spans, absorbed); +} + +/** Inside a dip that does not hold up as a whole, the music is out wherever it is absent for longer than a rest between notes. */ +function silencesIn(db: Float32Array, span: Span): Span[] { + const from = Math.round(span.start / HOP); + const to = Math.min(db.length, Math.round(span.end / HOP)); + const absent = new Uint8Array(db.length); + for (let f = from; f < to; f++) absent[f] = db[f]! < ABSENT_DB ? 1 : 0; + const shortSilence = (s: Span) => s.quiet && s.end - s.start < LONGEST_REST; + return candidates(absent, from, to, shortSilence).map((s) => + s.quiet && !holdsUp(db, s) ? { ...s, quiet: false } : s, + ); +} + +/** Breaks are where the music drops far under its running level, or out altogether, for half a second or more. */ +function runs(db: Float32Array): { segments: Run[]; breaks: Run[] } { + const quiet = new Uint8Array(db.length); + const heard: number[] = []; + for (let f = 0; f < db.length; f++) { + const level = db[f]!; + const reference = heard.length + ? median(heard.slice(-RUNNING_FRAMES)) + : Number.NEGATIVE_INFINITY; + quiet[f] = level < ABSENT_DB || level < reference - CUT_DB ? 1 : 0; + if (!quiet[f]) heard.push(level); + } + const merged = candidates(quiet, 0, db.length).flatMap((span) => + span.quiet && !holdsUp(db, span) ? silencesIn(db, span) : [span], + ); + const settled = joined(merged, (s) => !s.quiet && s.end - s.start < SHORTEST_SEGMENT); + return { + segments: settled.filter((s) => !s.quiet), + breaks: settled.filter((s) => s.quiet && s.start > 0 && s.end < db.length * HOP), + }; +} + +function chromaOf(chroma: Float64Array[], fps: number, run: Run): Float64Array { + const a = Math.floor(run.start * fps); + const b = Math.max(a + 1, Math.min(chroma.length, Math.ceil(run.end * fps))); + const sum = new Float64Array(12); + for (let f = a; f < b; f++) chroma[f]?.forEach((v, k) => (sum[k]! += v)); + const norm = Math.hypot(...sum); + return norm ? sum.map((v) => v / norm) : sum; +} + +const dot = (a: Float64Array, b: Float64Array) => a.reduce((n, v, k) => n + v * b[k]!, 0); + +function energyOf(samples: Float32Array, sampleRate: number, run: Run): number { + const a = Math.floor(run.start * sampleRate); + const b = Math.min(samples.length, Math.floor(run.end * sampleRate)); + let sum = 0; + for (let i = a; i < b; i++) sum += samples[i]! ** 2; + return b > a ? sum / (b - a) : 0; +} + +function shares(input: MusicInput, run: Run): Record { + const energies = Object.entries(input.stems).map( + ([name, samples]) => [name, energyOf(samples, input.sampleRate, run)] as const, + ); + const total = energies.reduce((n, [, e]) => n + e, 0); + return Object.fromEntries(energies.map(([name, e]) => [name, total ? e / total : 0])); +} + +function mixDistance(a: Record, b: Record): number { + return Object.keys(a).reduce((n, k) => n + Math.abs(a[k]! - (b[k] ?? 0)), 0) / 2; +} + +function bpmOf(beats: readonly number[], run: Run): number | null { + const inside = beats.filter((t) => t >= run.start && t < run.end); + const interval = inside.length >= 4 ? medianInterval(inside) : null; + return interval ? Math.round(60 / interval) : null; +} + +const sameTempo = (a: number | null, b: number | null) => + a === null || b === null || Math.abs(a - b) / Math.max(a, b) <= SAME_TEMPO; + +function fadeFrom(db: Float32Array, run: Run): number | null { + const a = Math.floor(run.start / HOP); + const b = Math.min(db.length, Math.floor(run.end / HOP)); + const body = median(db.subarray(a, b)); + const last = db[b - 1] ?? body; + if (body - last < FADE_DB) return null; + for (let f = Math.max(a, b - FADE_WINDOW / HOP); f < b; f++) + if (db[f]! < body - 3) return round(f * HOP, 1); + return null; +} + +type Known = Piece & { print: Float64Array; mix: Record }; + +interface Heard { + run: Run; + print: Float64Array; + mix: Record; + bpm: number | null; +} + +const alike = (p: Known, heard: Heard, piece: number, mix: number) => + dot(p.print, heard.print) >= piece && + mixDistance(p.mix, heard.mix) <= mix && + sameTempo(p.bpm, heard.bpm); + +/** The piece a part belongs to: the one before a short break, held loosely, else any heard before, held strictly. */ +function pieceOf( + pieces: Known[], + previous: MusicSegment | undefined, + heard: Heard, +): Known | undefined { + const last = previous && pieces.find((p) => p.id === previous.piece); + if ( + last && + heard.run.start - previous.end <= RESUMING_GAP && + alike(last, heard, RESUMING_PIECE, RESUMING_MIX) + ) + return last; + return pieces.find((p) => alike(p, heard, SAME_PIECE, SAME_MIX)); +} + +function newPiece(id: number, heard: Heard): Known { + return { + id, + instruments: Object.entries(heard.mix) + .filter(([, share]) => share >= LEADING_SHARE) + .sort((a, b) => b[1] - a[1]) + .map(([name]) => name), + bpm: heard.bpm, + firstAt: round(heard.run.start), + print: heard.print, + mix: heard.mix, + }; +} + +function energyCurve(melodic: Float32Array, sampleRate: number, duration: number) { + const energy = []; + for (let start = 0; start < duration; start += ENERGY_WINDOW) { + const end = Math.min(duration, start + ENERGY_WINDOW); + const power = energyOf(melodic, sampleRate, { start, end }); + energy.push({ + start, + end: round(end), + db: power > 0 ? Math.round(10 * Math.log10(power)) : -120, + }); + } + return energy; +} + +export function readMusic(input: MusicInput): { music: Music; beats: number[] } { + const melodic = mixed(Object.values(input.stems)); + const db = levels(melodic, input.sampleRate, HOP, LEVEL_WINDOW); + const { segments: found, breaks } = runs(db); + const frames = bandFramesOf(asSignal(melodic, input.sampleRate)); + const beats = rhythm( + asSignal(input.drums ? mixed([melodic, input.drums]) : melodic, input.sampleRate), + ).beats.times; + const pieces: Known[] = []; + const segments: MusicSegment[] = []; + for (const run of found) { + const heard: Heard = { + run, + print: chromaOf(frames.chroma, frames.chromaFps, run), + mix: shares(input, run), + bpm: bpmOf(beats, run), + }; + const previous = segments[segments.length - 1]; + let piece = pieceOf(pieces, previous, heard); + let entry: Entry; + if (piece) entry = previous?.piece === piece.id ? "resumes" : "returns"; + else { + piece = newPiece(pieces.length + 1, heard); + pieces.push(piece); + entry = previous ? "new" : "start"; + } + segments.push({ + start: round(run.start), + end: round(run.end), + piece: piece.id, + entry, + bpm: heard.bpm, + fadeFrom: fadeFrom(db, run), + }); + } + return { + music: { + segments, + pieces: pieces.map(({ print: _print, mix: _mix, ...piece }) => piece), + breaks: breaks.map((b) => ({ start: round(b.start), end: round(b.end) })), + energy: energyCurve(melodic, input.sampleRate, db.length * HOP), + }, + beats, + }; +} diff --git a/packages/cli/src/audio/read/pitch.ts b/packages/cli/src/audio/read/pitch.ts new file mode 100644 index 00000000000..ba4facec45d --- /dev/null +++ b/packages/cli/src/audio/read/pitch.ts @@ -0,0 +1,74 @@ +import { PITCH_RATE } from "./signal.js"; + +const FRAME = 320; +const HOP = 320; +const LOW_HZ = 70; +const HIGH_HZ = 500; +const VOICED_DB = -35; +const PERIODIC = 0.15; + +export interface PitchPoint { + t: number; + hz: number; +} + +const MIN_LAG = Math.floor(PITCH_RATE / HIGH_HZ); +const MAX_LAG = Math.ceil(PITCH_RATE / LOW_HZ); + +function voiced(pcm: Float32Array, start: number): boolean { + let energy = 0; + for (let i = 0; i < FRAME; i++) energy += pcm[start + i]! ** 2; + return 10 * Math.log10(energy / FRAME + 1e-12) >= VOICED_DB; +} + +function difference(pcm: Float32Array, start: number, tau: number): number { + let sum = 0; + for (let i = 0; i < FRAME; i++) { + const delta = pcm[start + i]! - pcm[start + i + tau]!; + sum += delta * delta; + } + return sum; +} + +/** The first lag whose normalized difference dips under the threshold, followed down to its minimum; -1 if none. */ +function periodOf(pcm: Float32Array, start: number, d: Float64Array): number { + let running = 0; + let lag = -1; + for (let tau = 1; tau <= MAX_LAG; tau++) { + const sum = difference(pcm, start, tau); + running += sum; + d[tau] = running > 0 ? (sum * tau) / running : 1; + if (lag === -1) { + if (tau >= MIN_LAG && d[tau]! < PERIODIC) lag = tau; + } else if (d[tau]! > d[tau - 1]!) break; + else if (d[tau]! < d[lag]!) lag = tau; + } + return lag; +} + +/** The fundamental of each voiced 40 ms frame of 8 kHz mono, by YIN's cumulative mean normalized difference. */ +export function pitchTrack(pcm: Float32Array): PitchPoint[] { + const out: PitchPoint[] = []; + const d = new Float64Array(MAX_LAG + 1); + for (let start = 0; start + FRAME + MAX_LAG <= pcm.length; start += HOP) { + if (!voiced(pcm, start)) continue; + const lag = periodOf(pcm, start, d); + if (lag > 0) out.push({ t: (start + FRAME / 2) / PITCH_RATE, hz: PITCH_RATE / lag }); + } + return out; +} + +export function within(track: readonly PitchPoint[], start: number, end: number): number[] { + let lo = 0; + let hi = track.length; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (track[mid]!.t < start) lo = mid + 1; + else hi = mid; + } + const out: number[] = []; + for (let i = lo; i < track.length && track[i]!.t < end; i++) out.push(track[i]!.hz); + return out; +} + +export const semitones = (a: number, b: number) => 12 * Math.log2(a / b); diff --git a/packages/cli/src/audio/read/read.test.ts b/packages/cli/src/audio/read/read.test.ts new file mode 100644 index 00000000000..3728b8af93d --- /dev/null +++ b/packages/cli/src/audio/read/read.test.ts @@ -0,0 +1,190 @@ +import { createServer, type IncomingMessage, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { readAudio } from "./read.js"; + +const RATE = 22_050; +const dir = mkdtempSync(join(tmpdir(), "hf-audio-read-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +function wav(name: string, seconds: number, fill: (t: number) => number): string { + const frames = Math.round(seconds * RATE); + const data = Buffer.alloc(frames * 2); + for (let i = 0; i < frames; i++) + data.writeInt16LE(Math.round(Math.max(-1, Math.min(1, fill(i / RATE))) * 32767), i * 2); + const header = Buffer.alloc(44); + header.write("RIFF", 0); + header.writeUInt32LE(36 + data.length, 4); + header.write("WAVEfmt ", 8); + header.writeUInt32LE(16, 16); + header.writeUInt16LE(1, 20); + header.writeUInt16LE(1, 22); + header.writeUInt32LE(RATE, 24); + header.writeUInt32LE(RATE * 2, 28); + header.writeUInt16LE(2, 32); + header.writeUInt16LE(16, 34); + header.write("data", 36); + header.writeUInt32LE(data.length, 40); + const path = join(dir, name); + writeFileSync(path, Buffer.concat([header, data])); + return path; +} + +const voiced = (t: number) => + (t >= 0.5 && t < 2) || (t >= 3 && t < 4.5) ? 0.3 * Math.sin(2 * Math.PI * 180 * t) : 0; +const boom = (t: number) => + t < 2.4 ? 0 : 0.7 * Math.exp(-(t - 2.4) / 0.3) * Math.sin(2 * Math.PI * 55 * (t - 2.4)); +const bed = (t: number) => + t > 5 && t < 6 + ? 0 + : 0.08 * Math.sin(2 * Math.PI * 261.63 * t) + 0.08 * Math.sin(2 * Math.PI * 392 * t); + +let server: Server; +let polls = 0; +let stemCalls = 0; +let stemRequest: unknown = null; +const stemFiles: Record = {}; + +async function body(req: IncomingMessage) { + const chunks: Uint8Array[] = []; + for await (const chunk of req) chunks.push(chunk); + return Buffer.concat(chunks); +} + +const origin = () => `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + +type Reply = (status: number, value: unknown) => void; + +const routes: Record void> = { + "POST /v3/assets/direct-uploads": (reply) => + reply(200, { + data: { asset_id: "episode", upload_url: `${origin()}/put`, upload_headers: {} }, + }), + "PUT /put": (reply) => reply(200, {}), + "POST /v3/assets/episode/complete": (reply) => reply(200, { data: { asset_id: "episode" } }), + "POST /v3/audio/stems": (reply, req, bytes) => { + expect(req.headers["x-api-key"]).toBe("test-key"); + stemCalls++; + stemRequest = JSON.parse(bytes.toString()); + reply(202, { data: { id: "job-1", status: "processing" } }); + }, + "GET /v3/audio/stems/job-1": (reply) => { + polls++; + reply(200, { + data: { + id: "job-1", + status: "completed", + stems: Object.keys(stemFiles).map((name) => ({ + name, + url: `${origin()}/stem/${name}.wav`, + })), + }, + }); + }, +}; + +beforeAll(async () => { + server = createServer(async (req, res) => { + const bytes = await body(req); + const reply: Reply = (status, value) => + res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(value)); + const route = routes[`${req.method} ${req.url}`]; + if (route) return route(reply, req, bytes); + const stem = req.url?.match(/^\/stem\/(\w+)\.wav$/)?.[1]; + if (stem && stemFiles[stem]) return res.writeHead(200).end(readFileSync(stemFiles[stem])); + res.writeHead(404).end(); + }); + await new Promise((done) => server.listen(0, "127.0.0.1", done)); +}); + +afterAll(() => new Promise((done) => server.close(() => done()))); + +afterEach(() => { + delete process.env.HYPERFRAMES_AUDIO_STEMS_URL; + delete process.env.HYPERFRAMES_AUDIO_API_KEY; +}); + +const ffmpeg = findFfBinary("ffmpeg") && findFfBinary("ffprobe"); + +describe.skipIf(!ffmpeg)("readAudio", () => { + it("reads the mix with its sidecar transcript when there are no stems, and reuses the cache", async () => { + const path = wav("talk.wav", 6, (t) => voiced(t)); + writeFileSync( + `${path}.transcript.json`, + JSON.stringify({ + audio: "talk.wav", + words: [ + { text: "一口气", start: 0.5, end: 1.4 }, + { text: "了解。", start: 1.4, end: 2 }, + { text: "伊朗", start: 3, end: 4.5 }, + ], + }), + ); + const cacheDir = join(dir, "cache-mix"); + const first = await readAudio(path, "talk.wav", { cacheDir }); + expect(first.cached).toBe(false); + expect(first.analysis).toMatchObject({ separation: "mix", stems: [], music: null }); + expect(first.analysis.voice!.sentences.map((s) => s.text)).toEqual(["一口气了解。", "伊朗"]); + expect(first.analysis.voice!.medianPitchHz).toBeGreaterThan(170); + expect(first.analysis.warnings.join("\n")).toContain("HYPERFRAMES_AUDIO_STEMS_URL is not set"); + const text = readFileSync(first.textPath, "utf8"); + expect(text).toContain('voice "一口气了解。"'); + expect(text).toContain("pause 1.0 s"); + const again = await readAudio(path, "talk.wav", { cacheDir }); + expect(again.cached).toBe(true); + writeFileSync( + `${path}.transcript.json`, + JSON.stringify({ words: [{ text: "换了", start: 0.5, end: 2 }] }), + ); + const edited = await readAudio(path, "talk.wav", { cacheDir }); + expect(edited.cached).toBe(false); + expect(edited.analysis.voice!.sentences[0]!.text).toBe("换了"); + }); + + it("separates through the stems service, then reads effects from the drums and the bed from the rest", async () => { + stemFiles.vocals = wav("vocals.wav", 8, voiced); + stemFiles.drums = wav("drums.wav", 8, boom); + stemFiles.piano = wav("piano.wav", 8, bed); + stemFiles.bass = wav("bass.wav", 8, () => 0); + const path = wav("episode.wav", 8, (t) => voiced(t) + boom(t) + bed(t)); + process.env.HYPERFRAMES_AUDIO_STEMS_URL = `${origin()}/v3/audio/stems`; + process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key"; + const cacheDir = join(dir, "cache-stems"); + const { analysis, textPath } = await readAudio(path, "episode.wav", { cacheDir }); + expect(stemRequest).toEqual({ version: 1, audio: { type: "asset_id", asset_id: "episode" } }); + expect(polls).toBe(1); + expect(analysis).toMatchObject({ + separation: "stems", + stems: ["bass", "drums", "piano", "vocals"], + warnings: [expect.stringContaining("no transcript")], + }); + expect(analysis.voice!.source).toBe("activity"); + expect(analysis.effects!.events.map((e) => [Math.round(e.t * 10) / 10, e.kind])).toEqual([ + [2.4, "boom"], + ]); + expect(analysis.effects!.events[0]!.overMusicDb).toBeGreaterThan(5); + expect(analysis.music!.breaks.map((b) => Math.round(b.start))).toEqual([5]); + expect(analysis.music!.segments.map((s) => s.entry)).toEqual(["start", "resumes"]); + expect(readFileSync(textPath, "utf8")).toContain("effect boom (S1): low"); + expect(existsSync(join(cacheDir, "stems"))).toBe(true); + await readAudio(path, "episode.wav", { cacheDir }); + expect(stemCalls).toBe(1); + }); + + it("keeps audio on the machine at quality local, and reads neither voice nor effects from a bare mix", async () => { + process.env.HYPERFRAMES_AUDIO_STEMS_URL = `${origin()}/v3/audio/stems`; + process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key"; + const path = wav("local.wav", 3, voiced); + const { analysis } = await readAudio(path, "local.wav", { + cacheDir: join(dir, "cache-local"), + quality: "local", + }); + expect(analysis).toMatchObject({ separation: "mix", voice: null, effects: null }); + expect(analysis.warnings[0]).toContain("quality local keeps audio on this machine"); + expect(analysis.warnings[1]).toContain("no transcript and no stems"); + }); +}); diff --git a/packages/cli/src/audio/read/read.ts b/packages/cli/src/audio/read/read.ts new file mode 100644 index 00000000000..7d317b732ec --- /dev/null +++ b/packages/cli/src/audio/read/read.ts @@ -0,0 +1,205 @@ +import { createHash } from "node:crypto"; +import { + createReadStream, + existsSync, + mkdirSync, + readFileSync, + renameSync, + writeFileSync, +} from "node:fs"; +import { basename, extname, join } from "node:path"; +import type { AnalysisQuality } from "@hyperframes/core/audio-analysis"; +import { loadTranscript } from "../../whisper/normalize.js"; +import { findEffects } from "./effects.js"; +import { formatRead } from "./format.js"; +import { readMusic } from "./music.js"; +import { pitchTrack } from "./pitch.js"; +import { PITCH_RATE, READ_RATE, decodeMono, mixed, round } from "./signal.js"; +import { DRUM_STEM, VOICE_STEM, cachedStems, fetchStems, stemsUnavailable } from "./stems.js"; +import { READ_VERSION, type ReadAudio } from "./types.js"; +import { voiceFromActivity, voiceFromWords, type TimedWord } from "./voice.js"; + +export interface ReadOptions { + /** Where analyses and stems are kept, normally the project's `.hyperframes/read-audio`. */ + cacheDir: string; + /** A transcript of the recording; `.transcript.json` next to it is used when absent. */ + transcript?: string; + quality?: AnalysisQuality; +} + +export interface ReadResult { + analysis: ReadAudio; + /** The whole recording as text, for grep. */ + textPath: string; + jsonPath: string; + cached: boolean; +} + +async function sha256(path: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer); + return hash.digest("hex"); +} + +function transcriptFor(path: string, given?: string): string | null { + if (given) return given; + const sidecar = `${path}.transcript.json`; + return existsSync(sidecar) ? sidecar : null; +} + +function wordsOf(transcript: string): TimedWord[] { + return loadTranscript(transcript).words.map(({ text, start, end }) => ({ text, start, end })); +} + +const fingerprint = (path: string | null) => + path ? createHash("sha256").update(readFileSync(path)).digest("hex").slice(0, 16) : "none"; + +function writeAtomic(path: string, content: string) { + const temporary = `${path}.${process.pid}.tmp`; + writeFileSync(temporary, content); + renameSync(temporary, path); +} + +interface CacheEntry { + analysis: ReadAudio; + transcriptKey: string; +} + +async function stemsFor( + path: string, + dir: string, + quality: AnalysisQuality, + warnings: string[], +): Promise | null> { + const cached = cachedStems(dir); + if (cached) return cached; + if (quality !== "best") { + warnings.push(`not separated into stems: quality ${quality} keeps audio on this machine`); + return null; + } + const missing = stemsUnavailable(); + if (missing) { + warnings.push(`not separated into stems: ${missing}`); + return null; + } + try { + return await fetchStems(path, dir); + } catch (error) { + warnings.push(`not separated into stems: ${error instanceof Error ? error.message : error}`); + return null; + } +} + +/** Effects from the drums and the music from the other non-voice stems, measured against the whole recording. */ +async function fromStems(path: string, stemPaths: Record) { + const pitched: Record = {}; + for (const [name, stemPath] of Object.entries(stemPaths)) + if (name !== VOICE_STEM && name !== DRUM_STEM) pitched[name] = await decodeMono(stemPath); + const drums = stemPaths[DRUM_STEM] ? await decodeMono(stemPaths[DRUM_STEM]) : null; + const bed = Object.keys(pitched).length ? mixed(Object.values(pitched)) : null; + const { music, beats } = bed + ? readMusic({ stems: pitched, drums, sampleRate: READ_RATE }) + : { music: null, beats: [] }; + const effects = findEffects({ + source: drums ?? bed ?? new Float32Array(0), + sampleRate: READ_RATE, + music: bed, + heard: await decodeMono(path), + beats, + speech: [], + }); + return { effects, music }; +} + +const transcribeHint = (file: string) => `run hyperframes transcribe ${file} and pass --transcript`; + +async function analyze( + path: string, + file: string, + sha: string, + transcript: string | null, + stemDir: string, + quality: AnalysisQuality, +): Promise { + const warnings: string[] = []; + const stemPaths = await stemsFor(path, stemDir, quality, warnings); + const words = transcript ? wordsOf(transcript) : []; + const voicePath = stemPaths?.[VOICE_STEM] ?? path; + const voiceSamples = await decodeMono(voicePath); + const about = base(file, sha, round(voiceSamples.length / READ_RATE, 3), stemPaths, transcript); + if (!stemPaths && !words.length) { + warnings.push( + `no transcript and no stems: voice and effects are not read, because the mix alone does not tell speech from music; ${transcribeHint(file)}`, + ); + return { ...about, voice: null, effects: null, music: null, warnings }; + } + if (!words.length) + warnings.push(`no transcript: voice lines carry no words; ${transcribeHint(file)}`); + const pitch = pitchTrack(await decodeMono(voicePath, PITCH_RATE)); + const voice = words.length + ? voiceFromWords(words, pitch) + : voiceFromActivity(voiceSamples, READ_RATE, pitch); + if (stemPaths) return { ...about, voice, ...(await fromStems(path, stemPaths)), warnings }; + warnings.push("without stems, effects under speech are not found and music is not read"); + const effects = findEffects({ + source: voiceSamples, + sampleRate: READ_RATE, + music: null, + heard: null, + beats: [], + speech: voice.sentences, + }); + return { ...about, voice, effects, music: null, warnings }; +} + +function base( + file: string, + sha: string, + duration: number, + stemPaths: Record | null, + transcript: string | null, +): Omit { + return { + version: READ_VERSION, + file, + sha256: sha, + duration, + separation: stemPaths ? "stems" : "mix", + stems: stemPaths ? Object.keys(stemPaths).sort() : [], + transcript: transcript ? basename(transcript) : null, + }; +} + +/** Read a recording as time-stamped voice, effect and music events, computed once per file content. */ +export async function readAudio( + path: string, + file: string, + options: ReadOptions, +): Promise { + const quality = options.quality ?? "best"; + const sha = await sha256(path); + const transcript = transcriptFor(path, options.transcript); + const transcriptKey = fingerprint(transcript); + const stem = `${basename(file, extname(file)).replace(/[^\w.-]/g, "_")}-${sha.slice(0, 12)}`; + mkdirSync(options.cacheDir, { recursive: true }); + const jsonPath = join(options.cacheDir, `${stem}.json`); + const textPath = join(options.cacheDir, `${stem}.txt`); + const stemDir = join(options.cacheDir, "stems", sha.slice(0, 12)); + if (existsSync(jsonPath) && existsSync(textPath)) { + const entry = JSON.parse(readFileSync(jsonPath, "utf8")) as CacheEntry; + const stemsNowPossible = + entry.analysis.separation === "mix" && + quality === "best" && + (cachedStems(stemDir) !== null || stemsUnavailable() === null); + if ( + entry.analysis.version === READ_VERSION && + entry.transcriptKey === transcriptKey && + !stemsNowPossible + ) + return { analysis: entry.analysis, textPath, jsonPath, cached: true }; + } + const analysis = await analyze(path, file, sha, transcript, stemDir, quality); + writeAtomic(jsonPath, JSON.stringify({ analysis, transcriptKey } satisfies CacheEntry)); + writeAtomic(textPath, formatRead(analysis)); + return { analysis, textPath, jsonPath, cached: false }; +} diff --git a/packages/cli/src/audio/read/signal.ts b/packages/cli/src/audio/read/signal.ts new file mode 100644 index 00000000000..837b1c49f46 --- /dev/null +++ b/packages/cli/src/audio/read/signal.ts @@ -0,0 +1,61 @@ +import type { AudioSignal } from "@hyperframes/core/audio-analysis"; +import { decodeAudio } from "../analysis/decode.js"; + +/** Every layer is read from mono at this rate: enough for onsets and band shapes, small enough for a long episode. */ +export const READ_RATE = 22050; +export const PITCH_RATE = 8000; +const SILENT_DB = -120; + +export async function decodeMono(path: string, sampleRate = READ_RATE): Promise { + const decoded = await decodeAudio(path, {}, undefined, { sampleRate, mono: true }); + if (!decoded) throw new Error(`${path} has no audio stream`); + return decoded.signal.channels[0]!; +} + +export const asSignal = (samples: Float32Array, sampleRate = READ_RATE): AudioSignal => ({ + sampleRate, + channels: [samples], +}); + +export function mixed(tracks: readonly Float32Array[]): Float32Array { + const length = Math.max(0, ...tracks.map((t) => t.length)); + const sum = new Float32Array(length); + for (const track of tracks) for (let i = 0; i < track.length; i++) sum[i]! += track[i]!; + return sum; +} + +/** RMS level in dBFS of each `hop`-second frame, measured over a `window` centred on it. */ +export function levels( + samples: Float32Array, + sampleRate: number, + hop: number, + window = hop * 2, +): Float32Array { + const hopSamples = Math.max(1, Math.round(hop * sampleRate)); + const half = Math.max(1, Math.round((window * sampleRate) / 2)); + const squares = new Float64Array(samples.length + 1); + for (let i = 0; i < samples.length; i++) squares[i + 1] = squares[i]! + samples[i]! ** 2; + const frames = Math.ceil(samples.length / hopSamples); + const out = new Float32Array(frames); + for (let f = 0; f < frames; f++) { + const center = f * hopSamples; + const a = Math.max(0, center - half); + const b = Math.min(samples.length, center + half); + const power = b > a ? (squares[b]! - squares[a]!) / (b - a) : 0; + out[f] = power > 0 ? Math.max(SILENT_DB, 10 * Math.log10(power)) : SILENT_DB; + } + return out; +} + +export function quantile(values: ArrayLike, q: number): number { + if (!values.length) return NaN; + const sorted = Float64Array.from(values).sort(); + return sorted[Math.min(sorted.length - 1, Math.max(0, Math.floor(q * (sorted.length - 1))))]!; +} + +export const median = (values: ArrayLike) => quantile(values, 0.5); + +export const round = (n: number, places = 2) => { + const k = 10 ** places; + return Math.round(n * k) / k; +}; diff --git a/packages/cli/src/audio/read/stems.ts b/packages/cli/src/audio/read/stems.ts new file mode 100644 index 00000000000..636dd0e7b0d --- /dev/null +++ b/packages/cli/src/audio/read/stems.ts @@ -0,0 +1,148 @@ +import { randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { extname, join } from "node:path"; +import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { HyperframesCloudClient } from "../../cloud/_gen/client.js"; +import { uploadAssetViaDirectUpload } from "../../cloud/upload.js"; +import { runFile, serviceConfig, serviceFailure } from "../analysis/providers/shared.js"; + +/** + * The stem separation service contract. + * + * Request: `POST ` with `x-api-key: `, a new + * `Idempotency-Key`, and `{ version: 1, audio: { type: "asset_id", asset_id } }`. The recording is uploaded + * first as 192 kb/s stereo MP3 through the same origin's asset direct-upload routes. + * + * Response: `{ data: StemsJob }`, or a bare `StemsJob`. While `status` is `pending` or `processing`, + * `GET /` returns the job again. A `completed` job lists one URL per stem; + * the names are vocals, drums, bass, guitar, piano and other. + */ +interface StemsJob { + id: string; + status: "pending" | "processing" | "completed" | "failed"; + stems?: { name: string; url: string }[]; + error?: { message?: string }; +} + +const URL_ENV = "HYPERFRAMES_AUDIO_STEMS_URL"; +const KEY_ENV = "HYPERFRAMES_AUDIO_API_KEY"; +const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS"; +const DEFAULT_TIMEOUT_MS = 30 * 60_000; +const POLL_MS = 5000; +const DONE = "stems.json"; + +export const VOICE_STEM = "vocals"; +export const DRUM_STEM = "drums"; + +const config = () => + serviceConfig({ url: URL_ENV, key: KEY_ENV, timeout: TIMEOUT_ENV }, DEFAULT_TIMEOUT_MS); + +/** Why stems cannot be fetched here, or null when they can. */ +export function stemsUnavailable(): string | null { + const { url, key } = config(); + if (!url) return `${URL_ENV} is not set`; + if (!key) return `${KEY_ENV} is not set`; + return null; +} + +/** Stems already fetched into `dir`, by name; null when there are none yet. */ +export function cachedStems(dir: string): Record | null { + const done = join(dir, DONE); + if (!existsSync(done)) return null; + const names = JSON.parse(readFileSync(done, "utf8")) as Record; + const paths = Object.fromEntries(Object.entries(names).map(([n, f]) => [n, join(dir, f)])); + return Object.values(paths).every((p) => existsSync(p)) ? paths : null; +} + +async function job(response: Response): Promise { + if (!response.ok) throw await serviceFailure(response, "the stems service"); + const body = (await response.json()) as { data?: StemsJob } | StemsJob | null; + const value = (body && "data" in body ? body.data : body) as StemsJob | undefined; + if (!value?.id || !value.status) throw new Error("the stems service answered without a job"); + return value; +} + +async function encoded(path: string, dir: string): Promise { + const ffmpeg = findFfBinary("ffmpeg", { configuredMustExist: true }); + if (!ffmpeg) throw new Error("ffmpeg is required to separate stems; run: npx hyperframes doctor"); + const out = join(dir, "upload.mp3"); + await runFile( + ffmpeg, + ["-v", "error", "-y", "-i", path, "-vn", "-ac", "2", "-ar", "44100", "-b:a", "192k", out], + { windowsHide: true, maxBuffer: 8 * 1024 * 1024 }, + ); + return out; +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Separate the recording through the configured service and save each stem into `dir`. */ +export async function fetchStems(path: string, dir: string): Promise> { + const { url, key, timeoutMs } = config(); + if (!url || !key) throw new Error(stemsUnavailable()!); + const deadline = Date.now() + timeoutMs; + const headers = { "x-api-key": key }; + const scratch = await mkdtemp(join(tmpdir(), "hf-stems-")); + try { + const mp3 = await encoded(path, scratch); + const client = new HyperframesCloudClient({ + baseUrl: new URL(url).origin, + getAuthHeaders: () => headers, + fetchImpl: (input, init) => fetch(input, { ...init, redirect: "error" }), + }); + const asset = await uploadAssetViaDirectUpload({ + client, + path: mp3, + size: statSync(mp3).size, + filename: "audio.mp3", + contentType: "audio/mpeg", + idempotencyKey: randomUUID(), + }); + let current = await job( + await fetch(url, { + method: "POST", + redirect: "error", + headers: { + ...headers, + "idempotency-key": randomUUID(), + "content-type": "application/json", + }, + body: JSON.stringify({ version: 1, audio: { type: "asset_id", asset_id: asset.asset_id } }), + }), + ); + while (current.status === "pending" || current.status === "processing") { + if (Date.now() > deadline) + throw new Error(`stem separation did not finish within ${Math.round(timeoutMs / 1000)} s`); + await sleep(POLL_MS); + current = await job( + await fetch(`${url.replace(/\/$/, "")}/${encodeURIComponent(current.id)}`, { + redirect: "error", + headers, + }), + ); + } + if (current.status !== "completed") + throw new Error( + `stem separation failed${current.error?.message ? `: ${current.error.message}` : ""}`, + ); + const stems = current.stems ?? []; + if (!stems.some((s) => s.name === VOICE_STEM)) + throw new Error("the stems service returned no vocals stem"); + mkdirSync(dir, { recursive: true }); + const names: Record = {}; + for (const stem of stems) { + const response = await fetch(stem.url); + if (!response.ok) + throw new Error(`could not download the ${stem.name} stem: ${response.status}`); + const file = `${stem.name.replace(/[^\w-]/g, "_")}${extname(new URL(stem.url).pathname) || ".mp3"}`; + writeFileSync(join(dir, file), Buffer.from(await response.arrayBuffer())); + names[stem.name] = file; + } + writeFileSync(join(dir, DONE), JSON.stringify(names)); + return cachedStems(dir)!; + } finally { + await rm(scratch, { recursive: true, force: true }); + } +} diff --git a/packages/cli/src/audio/read/types.ts b/packages/cli/src/audio/read/types.ts new file mode 100644 index 00000000000..608a2db5002 --- /dev/null +++ b/packages/cli/src/audio/read/types.ts @@ -0,0 +1,104 @@ +import type { Span } from "@hyperframes/core/audio-analysis"; + +export const READ_VERSION = 1; + +export type Separation = "stems" | "mix"; + +export type RateUnit = "chars" | "words"; + +export type Ending = "rising" | "falling" | "level"; + +export interface Sentence { + start: number; + end: number; + /** Null when there is no transcript and the span comes from where the voice is active. */ + text: string | null; + rate: number | null; + pitchHz: number | null; + ending: Ending | null; +} + +export interface Voice { + source: "transcript" | "activity"; + unit: RateUnit; + sentences: Sentence[]; + pauses: Span[]; + meanRate: number | null; + medianPitchHz: number | null; +} + +export type EffectKind = "riser" | "boom" | "thump" | "whoosh" | "click" | "tick" | "ding" | "hit"; + +export type Band = "low" | "mid" | "high"; + +export interface Effect { + t: number; + end: number; + kind: EffectKind; + group: string; + band: Band; + attackMs: number; + tail: number; + /** How far the effect rises out of what was playing just before it. */ + riseDb: number; + /** The effect's peak against the music at that moment; null without a music stem or while the music is out. */ + overMusicDb: number | null; + /** More than one when a quick run of the same sound is reported as one train. */ + count: number; +} + +export interface EffectGroup { + id: string; + kind: EffectKind; + count: number; +} + +export interface Effects { + events: Effect[]; + groups: EffectGroup[]; + /** Hits that repeat on the music's beat, which are the music's own drums and not listed as effects. */ + drumHits: number; +} + +export type Entry = "start" | "resumes" | "returns" | "new"; + +export interface MusicSegment { + start: number; + end: number; + piece: number; + entry: Entry; + bpm: number | null; + /** Where a fade out begins, when the segment ends on one. */ + fadeFrom: number | null; +} + +export interface Piece { + id: number; + /** The stems that carry it, loudest first. */ + instruments: string[]; + bpm: number | null; + firstAt: number; +} + +export interface Music { + segments: MusicSegment[]; + pieces: Piece[]; + breaks: Span[]; + /** Level of the music every 30 s, in dBFS. */ + energy: { start: number; end: number; db: number }[]; +} + +export interface ReadAudio { + version: typeof READ_VERSION; + file: string; + sha256: string; + duration: number; + separation: Separation; + stems: string[]; + transcript: string | null; + /** Null when the mix alone cannot tell speech from music: no stems and no transcript. */ + voice: Voice | null; + effects: Effects | null; + music: Music | null; + warnings: string[]; +} diff --git a/packages/cli/src/audio/read/voice.test.ts b/packages/cli/src/audio/read/voice.test.ts new file mode 100644 index 00000000000..901bca76373 --- /dev/null +++ b/packages/cli/src/audio/read/voice.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { voiceFromActivity, voiceFromWords } from "./voice.js"; +import { READ_RATE } from "./signal.js"; + +const glide = (start: number, end: number, from: number, to: number, step = 0.04) => { + const points = []; + for (let t = start; t < end; t += step) + points.push({ t, hz: from + ((to - from) * (t - start)) / (end - start) }); + return points; +}; + +describe("voiceFromWords", () => { + const words = [ + { text: "伊", start: 1.0, end: 1.2 }, + { text: "朗", start: 1.2, end: 1.4 }, + { text: "经", start: 1.4, end: 1.6 }, + { text: "济。", start: 1.6, end: 2.0 }, + { text: "涨", start: 3.0, end: 3.25 }, + { text: "了", start: 3.25, end: 3.5 }, + ]; + + it("splits sentences at end punctuation, counts Han characters per second and lists pauses", () => { + const voice = voiceFromWords(words, []); + expect(voice.unit).toBe("chars"); + expect(voice.sentences.map((s) => [s.text, s.start, s.end, s.rate])).toEqual([ + ["伊朗经济。", 1, 2, 4], + ["涨了", 3, 3.5, 4], + ]); + expect(voice.pauses).toEqual([{ start: 2, end: 3 }]); + expect(voice.meanRate).toBe(4); + }); + + it("says a sentence ends rising when its last third lifts two semitones or more", () => { + const track = [...glide(1, 1.7, 180, 180), ...glide(1.7, 2, 180, 240)]; + const voice = voiceFromWords(words, track); + expect(voice.sentences[0]).toMatchObject({ ending: "rising", pitchHz: 180 }); + expect(voice.sentences[1]!.ending).toBeNull(); + }); + + it("counts words, not letters, in a Latin transcript", () => { + const voice = voiceFromWords( + [ + { text: "Prices", start: 0, end: 0.5 }, + { text: "rose", start: 0.5, end: 0.8 }, + { text: "fast.", start: 0.8, end: 1 }, + ], + [], + ); + expect(voice.unit).toBe("words"); + expect(voice.sentences[0]).toMatchObject({ text: "Prices rose fast.", rate: 3 }); + }); +}); + +describe("voiceFromActivity", () => { + it("finds the voiced spans and the pause between them without a transcript", () => { + const samples = new Float32Array(4 * READ_RATE); + for (let i = 0; i < samples.length; i++) { + const t = i / READ_RATE; + if ((t >= 0.5 && t < 1.5) || (t >= 2.5 && t < 3.5)) + samples[i] = 0.3 * Math.sin(2 * Math.PI * 200 * t); + } + const voice = voiceFromActivity(samples, READ_RATE, []); + expect(voice.source).toBe("activity"); + expect(voice.sentences.map((s) => [s.text, Math.round(s.start * 10) / 10])).toEqual([ + [null, 0.5], + [null, 2.5], + ]); + expect(voice.pauses).toHaveLength(1); + expect(voice.pauses[0]!.end - voice.pauses[0]!.start).toBeCloseTo(1, 0); + }); +}); diff --git a/packages/cli/src/audio/read/voice.ts b/packages/cli/src/audio/read/voice.ts new file mode 100644 index 00000000000..ec995289254 --- /dev/null +++ b/packages/cli/src/audio/read/voice.ts @@ -0,0 +1,162 @@ +import type { Span } from "@hyperframes/core/audio-analysis"; +import { median, round, levels, quantile } from "./signal.js"; +import { semitones, within, type PitchPoint } from "./pitch.js"; +import type { Ending, RateUnit, Sentence, Voice } from "./types.js"; + +export interface TimedWord { + text: string; + start: number; + end: number; +} + +const SENTENCE_END = /[。！？!?…；;.]["'”’」』)]*$/; +const SENTENCE_GAP = 0.6; +const LONGEST_SENTENCE = 60; +const PAUSE_SECONDS = 0.4; +const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu; +const LETTERS = /\p{L}/u; +const ENDING_SEMITONES = 2; +const MIN_PITCH_FRAMES = 5; + +function rateUnit(words: readonly TimedWord[]): RateUnit { + const text = words.map((w) => w.text).join(""); + return (text.match(CJK)?.length ?? 0) * 2 >= text.replace(/\s/g, "").length ? "chars" : "words"; +} + +function spoken(words: readonly TimedWord[], unit: RateUnit): number { + if (unit === "chars") return words.reduce((n, w) => n + (w.text.match(CJK)?.length ?? 0), 0); + return words.filter((w) => LETTERS.test(w.text)).length; +} + +function joined(words: readonly TimedWord[], unit: RateUnit): string { + return unit === "chars" + ? words.map((w) => w.text.trim()).join("") + : words + .map((w) => w.text.trim()) + .join(" ") + .replace(/\s+([,.!?;:])/g, "$1"); +} + +function sentenceGroups(words: readonly TimedWord[]): TimedWord[][] { + const groups: TimedWord[][] = []; + let current: TimedWord[] = []; + let chars = 0; + words.forEach((word, i) => { + current.push(word); + chars += word.text.length; + const next = words[i + 1]; + if ( + !next || + SENTENCE_END.test(word.text.trim()) || + next.start - word.end >= SENTENCE_GAP || + chars >= LONGEST_SENTENCE + ) { + groups.push(current); + current = []; + chars = 0; + } + }); + return groups; +} + +function ending(track: readonly PitchPoint[], start: number, end: number): Ending | null { + const split = start + (end - start) * 0.7; + const body = within(track, start, split); + const tail = within(track, split, end); + if (body.length < MIN_PITCH_FRAMES || tail.length < 2) return null; + const shift = semitones(median(tail), median(body)); + if (shift >= ENDING_SEMITONES) return "rising"; + if (shift <= -ENDING_SEMITONES) return "falling"; + return "level"; +} + +function pitchOf(track: readonly PitchPoint[], start: number, end: number): number | null { + const hz = within(track, start, end); + return hz.length >= MIN_PITCH_FRAMES ? Math.round(median(hz)) : null; +} + +function summary(sentences: Sentence[], pauses: Span[], track: readonly PitchPoint[]) { + const timed = sentences.filter((s) => s.rate !== null); + const seconds = timed.reduce((n, s) => n + (s.end - s.start), 0); + const meanRate = seconds + ? round(timed.reduce((n, s) => n + s.rate! * (s.end - s.start), 0) / seconds, 1) + : null; + return { + sentences, + pauses, + meanRate, + medianPitchHz: track.length ? Math.round(median(track.map((p) => p.hz))) : null, + }; +} + +export function voiceFromWords(words: readonly TimedWord[], track: readonly PitchPoint[]): Voice { + const ordered = words.filter((w) => w.end > w.start).sort((a, b) => a.start - b.start); + const unit = rateUnit(ordered); + const sentences = sentenceGroups(ordered).map((group): Sentence => { + const start = group[0]!.start; + const end = group[group.length - 1]!.end; + const count = spoken(group, unit); + return { + start: round(start), + end: round(end), + text: joined(group, unit), + rate: count && end > start ? round(count / (end - start), 1) : null, + pitchHz: pitchOf(track, start, end), + ending: ending(track, start, end), + }; + }); + const pauses: Span[] = []; + for (let i = 1; i < ordered.length; i++) { + const gap = ordered[i]!.start - ordered[i - 1]!.end; + if (gap >= PAUSE_SECONDS) + pauses.push({ start: round(ordered[i - 1]!.end), end: round(ordered[i]!.start) }); + } + return { source: "transcript", unit, ...summary(sentences, pauses, track) }; +} + +const ACTIVITY_HOP = 0.05; +const SHORTEST_SPEECH = 0.3; + +/** Where the voice is active, for a recording without a transcript: spans between pauses, without words. */ +export function voiceFromActivity( + vocals: Float32Array, + sampleRate: number, + track: readonly PitchPoint[], +): Voice { + const db = levels(vocals, sampleRate, ACTIVITY_HOP); + const threshold = Math.max(-45, quantile(db, 0.9) - 30); + const spans: Span[] = []; + let start: number | null = null; + let quietSince: number | null = null; + db.forEach((level, f) => { + const t = f * ACTIVITY_HOP; + if (level >= threshold) { + if (start === null) start = t; + quietSince = null; + } else if (start !== null) { + quietSince ??= t; + if (t - quietSince >= PAUSE_SECONDS) { + if (quietSince - start >= SHORTEST_SPEECH) spans.push({ start, end: quietSince }); + start = null; + quietSince = null; + } + } + }); + const end = db.length * ACTIVITY_HOP; + if (start !== null && (quietSince ?? end) - start >= SHORTEST_SPEECH) + spans.push({ start, end: quietSince ?? end }); + const sentences = spans.map( + (s): Sentence => ({ + start: round(s.start), + end: round(s.end), + text: null, + rate: null, + pitchHz: pitchOf(track, s.start, s.end), + ending: ending(track, s.start, s.end), + }), + ); + const pauses = spans + .slice(1) + .map((s, i) => ({ start: round(spans[i]!.end), end: round(s.start) })); + return { source: "activity", unit: "chars", ...summary(sentences, pauses, track) }; +} diff --git a/packages/cli/src/commands/audio.ts b/packages/cli/src/commands/audio.ts index 85cc7900a57..9541eda8538 100644 --- a/packages/cli/src/commands/audio.ts +++ b/packages/cli/src/commands/audio.ts @@ -1,4 +1,4 @@ -import { resolve } from "node:path"; +import { join, resolve } from "node:path"; import { defineCommand } from "citty"; import type { AnalysisQuality, @@ -10,6 +10,8 @@ import { DEFAULT_CAPABILITIES, type AnalyzeOptions } from "../audio/analysis/ana import { formatReport } from "../audio/analysis/format.js"; import { analyzeTargets, previewTarget } from "../audio/analysis/run.js"; import { filmTargets, type AudioTarget, type FilmQuery } from "../audio/analysis/targets.js"; +import { formatRead } from "../audio/read/format.js"; +import { readAudio } from "../audio/read/read.js"; import { describeProject } from "../timeline/describeProject.js"; import { CliUsageError } from "../utils/commandResult.js"; import { ensureDOMParser } from "../utils/dom.js"; @@ -29,6 +31,10 @@ export const examples: Example[] = [ "The beat grid, tempo and sections of a music bed", "hyperframes audio music.mp3 --only beats,downbeats,bpm,sections", ], + [ + "A whole recording as voice, effect and music lines, 60–70 s shown", + "hyperframes audio refs/episode.mp4 --read --range 60,70", + ], [ "Only onsets, from 60–200 Hz, finer", "hyperframes audio drums.wav --only onsets --band 60,200 --hop-ms 2", @@ -114,6 +120,9 @@ interface AudioFlags { "onset-ratio"?: string; "tail-db"?: string; "silence-db"?: string; + read?: boolean; + transcript?: string; + json?: boolean; } function analyzeRequest(flags: AudioFlags): AnalyzeOptions { @@ -160,6 +169,30 @@ async function clipTargets(flags: AudioFlags, query: FilmQuery | null): Promise< return filmTargets(project.dir, timeline, query ?? { by: "all" }); } +function readQuality(files: string[], flags: AudioFlags): AnalysisQuality { + if (files.length !== 1) throw new CliUsageError("--read reads exactly one file"); + if (flags.element || flags.at || flags["place-at"]) + throw new CliUsageError("--read takes a file and --range, not a film query"); + const quality = (flags.quality ?? "best") as AnalysisQuality; + if (!QUALITIES.has(quality)) throw new CliUsageError("--quality is best, local or fast"); + return quality; +} + +async function read(files: string[], flags: AudioFlags) { + const quality = readQuality(files, flags); + const range = pair("range", flags.range); + const result = await readAudio(resolve(process.cwd(), files[0]!), files[0]!, { + cacheDir: join(resolve(flags.dir ?? process.cwd()), ".hyperframes", "read-audio"), + transcript: flags.transcript ? resolve(process.cwd(), flags.transcript) : undefined, + quality, + }); + if (flags.json) { + console.log(JSON.stringify(withMeta(result))); + return; + } + console.log(`${formatRead(result.analysis, range)}\n\nwhole recording: ${result.textPath}`); +} + export default defineCommand({ meta: { name: "audio", @@ -212,6 +245,16 @@ export default defineCommand({ description: "Include each capability's provider ladder", default: false, }, + read: { + type: "boolean", + description: + "Read one whole recording as voice, effect and music lines (--range picks what is shown)", + default: false, + }, + transcript: { + type: "string", + description: "With --read: the recording's transcript (default .transcript.json)", + }, json: { type: "boolean", description: "Output as JSON", default: false }, }, setup({ rawArgs, cmd }) { @@ -220,6 +263,8 @@ export default defineCommand({ async run({ args }) { const flags = args as unknown as AudioFlags; const files = args._.map(String); + if (flags.read) return read(files, flags); + if (flags.transcript) throw new CliUsageError("--transcript only works with --read"); const query = filmQuery(flags); if (files.length && query) throw new CliUsageError("give files or a film query, not both"); const targets = files.length From 37c215812bfe64b3adb251fc240f5872b8e69f20 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:09:44 -0700 Subject: [PATCH 02/12] refactor(cli): name the service error parse instead of commenting it --- packages/cli/src/audio/analysis/decode.ts | 1 - .../cli/src/audio/analysis/providers/shared.ts | 15 ++++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/audio/analysis/decode.ts b/packages/cli/src/audio/analysis/decode.ts index 00fe4a4056c..9ba709330a3 100644 --- a/packages/cli/src/audio/analysis/decode.ts +++ b/packages/cli/src/audio/analysis/decode.ts @@ -55,7 +55,6 @@ export interface DecodeWindow { seconds?: number; } -/** Decode to this rate, and to one channel, instead of the file's own layout. */ export interface DecodeFormat { sampleRate?: number; mono?: boolean; diff --git a/packages/cli/src/audio/analysis/providers/shared.ts b/packages/cli/src/audio/analysis/providers/shared.ts index 689a9e3b400..5e78700bc19 100644 --- a/packages/cli/src/audio/analysis/providers/shared.ts +++ b/packages/cli/src/audio/analysis/providers/shared.ts @@ -75,7 +75,6 @@ export function downbeatsWithMeter(beats: number[], downbeats: number[]): Downbe const round = (t: number) => Math.round(t * 1000) / 1000; export const rounded = (times: readonly number[]) => times.map(round); -/** A remote service's URL, key and timeout, read from the environment each call. */ export function serviceConfig( env: { url: string; key: string; timeout: string }, defaultMs: number, @@ -88,14 +87,16 @@ export function serviceConfig( }; } -/** A failed response as an error carrying the service's own message when it sent one. */ -export async function serviceFailure(response: Response, service: string): Promise { - const text = await response.text().catch(() => ""); - let detail = text.slice(0, 200); +function errorMessageIn(text: string): string | undefined { try { - detail = (JSON.parse(text) as { error?: { message?: string } }).error?.message ?? detail; + return (JSON.parse(text) as { error?: { message?: string } }).error?.message; } catch { - // not JSON: keep the text + return undefined; } +} + +export async function serviceFailure(response: Response, service: string): Promise { + const text = await response.text().catch(() => ""); + const detail = errorMessageIn(text) ?? text.slice(0, 200); return new Error(`${service} answered ${response.status}${detail ? `: ${detail}` : ""}`); } From 299c0c067194a4fa91a6b31e9092de302c9f9571 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:41:39 -0700 Subject: [PATCH 03/12] fix(cli): call HeyGen's audio services with the CLI's HeyGen credential Stems for `audio --read` and the `api` analysis provider only ran when HYPERFRAMES_AUDIO_STEMS_URL / HYPERFRAMES_AUDIO_API_URL and HYPERFRAMES_AUDIO_API_KEY were set, so someone with HEYGEN_API_KEY or a saved `hyperframes auth login --api-key` still got "not set" and had to know the internal route. Both now default to /v3/audio/stems and /v3/audio/analysis on the HeyGen API (HEYGEN_API_URL when set) with the credential the other HeyGen commands use. The URL variables still point at another service, which then needs its own HYPERFRAMES_AUDIO_API_KEY, so the HeyGen credential never goes to a host it was not issued for. --- docs/packages/cli.mdx | 34 +++++++++---- .../src/audio/analysis/providers/api.test.ts | 40 ++++++++++++--- .../cli/src/audio/analysis/providers/api.ts | 42 +++++++++------- .../src/audio/analysis/providers/shared.ts | 50 ++++++++++++++++--- packages/cli/src/audio/read/read.test.ts | 16 ++++-- packages/cli/src/audio/read/read.ts | 12 ++--- packages/cli/src/audio/read/stems.ts | 40 ++++++++------- 7 files changed, 163 insertions(+), 71 deletions(-) diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index adf91ad3f8b..1c707c94d2d 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -611,11 +611,12 @@ Sentences come from a transcript: `--transcript `, or carry no words. Effects and music are read from stems. At `--quality best` (the default) the -recording is separated into vocals, drums, bass, guitar, piano and other by the -service at `HYPERFRAMES_AUDIO_STEMS_URL`, with `HYPERFRAMES_AUDIO_API_KEY`: it -is uploaded as 192 kb/s MP3 through the same origin's direct-upload endpoints, -`POST` starts a job, and `GET /` is polled until the stems are ready -(`HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS`, 30 minutes by default). Effects are read +recording is separated into vocals, drums, bass, guitar, piano and other by +HeyGen's `/v3/audio/stems`, with your HeyGen credential (see +[audio services](#audio-services)): it is uploaded as 192 kb/s MP3 through the +same origin's direct-upload endpoints, `POST` starts a job, and `GET /` +is polled until the stems are ready (`HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS`, 30 +minutes by default). Effects are read from the drums stem and the music from the other four. Without stems, effects are read from the mix outside speech and the music is not read; with neither stems nor a transcript, voice and effects are not read either, because the mix alone does @@ -635,13 +636,12 @@ answers: | sections | `api`, `apple-mu`, `signal` | | describe | `api`, `apple-sound` | -- `api` is an audio analysis service, used when `HYPERFRAMES_AUDIO_API_URL` and - `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded as 16 kHz mono +- `api` is HeyGen's `/v3/audio/analysis`, used when a HeyGen credential is + found (see [audio services](#audio-services)). The audio is uploaded as 16 kHz mono WAV through the same origin’s public asset direct-upload and completion endpoints. Each decoded signal is uploaded once; analysis requests send its finalized asset ID as JSON. - The API key needs asset upload and audio-analysis permissions. Channels are averaged unless cancellation removes over 90% of the loudest - channel’s energy, in which case that channel is used. Requests carry the key in an `x-api-key` header and a new `Idempotency-Key` on each - request; the answer may come wrapped in `{ "data": … }`. Every beat, + The credential needs asset upload and audio permissions. Channels are averaged unless cancellation removes over 90% of the loudest + channel’s energy, in which case that channel is used. Each request carries a new `Idempotency-Key`; the answer may come wrapped in `{ "data": … }`. Every beat, downbeat or section boundary it answers moves onto the nearest beat signal processing measured, when one is within a quarter beat; the rest are listed in `snap.unsnapped`, and `snap.medianShiftMs` says how far the moved ones went. @@ -670,6 +670,20 @@ Tune the analysis with `--onset-method`, `--threshold`, `--band lo,hi`, `--hop-ms`, `--onset-ratio`, `--tail-db` and `--silence-db`. Decoding needs FFmpeg. +#### Audio services + +Stems and the `api` provider call HeyGen with the same credential as the other +HeyGen commands: `HEYGEN_API_KEY`, `HYPERFRAMES_API_KEY`, or one saved by +`hyperframes auth login --api-key`, on the API at `HEYGEN_API_URL` when set. +These routes take an API key; a `hyperframes auth login` browser sign-in is +refused, and the analysis falls back with a warning. Without a credential, +stems are skipped and `api` is passed over. + +To use another service, set `HYPERFRAMES_AUDIO_STEMS_URL` or +`HYPERFRAMES_AUDIO_API_URL` to its full URL together with +`HYPERFRAMES_AUDIO_API_KEY`, which is sent as `x-api-key`. Your HeyGen +credential is never sent to a URL set this way. + ## Look at it ### `normalize-audio` diff --git a/packages/cli/src/audio/analysis/providers/api.test.ts b/packages/cli/src/audio/analysis/providers/api.test.ts index 09f5067ce58..2444f9300e0 100644 --- a/packages/cli/src/audio/analysis/providers/api.test.ts +++ b/packages/cli/src/audio/analysis/providers/api.test.ts @@ -1,6 +1,6 @@ import { createServer, type IncomingMessage, type Server } from "node:http"; import type { AddressInfo } from "node:net"; -import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { rhythm, signalProvider, @@ -28,6 +28,7 @@ function clicks(seconds: number): AudioSignal { interface Received { method: string; + url: string | undefined; apiKey: string | undefined; authorization: string | undefined; idempotencyKey: string | undefined; @@ -94,6 +95,7 @@ beforeAll(async () => { }; received.push({ method: req.method!, + url: req.url, apiKey: req.headers["x-api-key"] as string | undefined, authorization: req.headers.authorization, idempotencyKey: req.headers["idempotency-key"] as string | undefined, @@ -108,32 +110,54 @@ beforeAll(async () => { afterAll(() => new Promise((done) => server.close(() => done()))); +const serverOrigin = () => `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + +beforeEach(() => { + for (const name of ["HEYGEN_API_KEY", "HYPERFRAMES_API_KEY", "HEYGEN_ACCESS_TOKEN"]) + vi.stubEnv(name, ""); + vi.stubEnv("HEYGEN_API_URL", serverOrigin()); +}); + afterEach(() => { received = []; uploads = 0; enveloped = true; + vi.unstubAllEnvs(); delete process.env.HYPERFRAMES_AUDIO_API_URL; delete process.env.HYPERFRAMES_AUDIO_API_KEY; }); function configure() { - process.env.HYPERFRAMES_AUDIO_API_URL = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/audio`; + process.env.HYPERFRAMES_AUDIO_API_URL = `${serverOrigin()}/v1/audio`; process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key"; } describe("apiProvider", () => { - it("is unavailable until both the URL and the key are set", () => { - expect(apiProvider.available!()).toEqual({ + it("is unavailable without a HeyGen credential", async () => { + expect(await apiProvider.available!()).toEqual({ ok: false, - reason: "HYPERFRAMES_AUDIO_API_URL is not set", + reason: expect.stringContaining("no HeyGen credential"), }); + vi.stubEnv("HEYGEN_API_KEY", "test-key"); + expect(await apiProvider.available!()).toEqual({ ok: true }); + }); + + it("calls HeyGen's audio analysis route with the CLI's HeyGen API key", async () => { + vi.stubEnv("HEYGEN_API_KEY", "test-key"); + reply = () => ({ status: 200, body: { version: 1, beats: { times: [0.25, 0.75] } } }); + await apiProvider.analyze("beats", clicks(2), {}); + expect(received.map((r) => [r.url, r.apiKey])).toEqual([["/v3/audio/analysis", "test-key"]]); + }); + + it("never sends the HeyGen credential to a service named by HYPERFRAMES_AUDIO_API_URL", async () => { + vi.stubEnv("HEYGEN_API_KEY", "heygen-key"); process.env.HYPERFRAMES_AUDIO_API_URL = "http://127.0.0.1:1/"; - expect(apiProvider.available!()).toEqual({ + expect(await apiProvider.available!()).toEqual({ ok: false, - reason: "HYPERFRAMES_AUDIO_API_KEY is not set", + reason: "HYPERFRAMES_AUDIO_API_URL is set but HYPERFRAMES_AUDIO_API_KEY is not", }); process.env.HYPERFRAMES_AUDIO_API_KEY = "k"; - expect(apiProvider.available!()).toEqual({ ok: true }); + expect(await apiProvider.available!()).toEqual({ ok: true }); }); it("refuses redirects without sending the API key to another origin", async () => { diff --git a/packages/cli/src/audio/analysis/providers/api.ts b/packages/cli/src/audio/analysis/providers/api.ts index ddf1e65a7b0..656f123b505 100644 --- a/packages/cli/src/audio/analysis/providers/api.ts +++ b/packages/cli/src/audio/analysis/providers/api.ts @@ -15,16 +15,18 @@ import { encodeWav, perSignal, rounded, - serviceConfig, + serviceAccess, serviceFailure, + type ServiceAccess, unavailable, } from "./shared.js"; /** * The audio analysis service contract. * - * Request: `POST `, exactly as configured, with `x-api-key: `, - * a new `Idempotency-Key` per request, and an {@link AudioApiRequest} JSON body. + * Request: `POST /v3/audio/analysis` on the HeyGen API with the CLI's HeyGen credential, or `POST + * ` exactly as configured with `x-api-key: `; a new + * `Idempotency-Key` per request, and an {@link AudioApiRequest} JSON body. * The analyzed span is uploaded once as 16-bit PCM WAV, mono at 16 kHz, using the same origin * public asset direct-upload and completion routes. The body references that finalized asset. * @@ -123,8 +125,18 @@ function uploaded(signal: AudioSignal): AudioSignal { const askedTogether = (capability: AudioApiCapability): AudioApiCapability[] => capability === "beats" || capability === "downbeats" ? ["beats", "downbeats"] : [capability]; -const config = () => - serviceConfig({ url: URL_ENV, key: KEY_ENV, timeout: TIMEOUT_ENV }, DEFAULT_TIMEOUT_MS); +const access = () => + serviceAccess( + { url: URL_ENV, key: KEY_ENV, timeout: TIMEOUT_ENV }, + "/v3/audio/analysis", + DEFAULT_TIMEOUT_MS, + ); + +async function granted(): Promise { + const found = await access(); + if (typeof found === "string") throw new Error(found); + return found; +} const isNumberList = (value: unknown): value is number[] => Array.isArray(value) && value.every((t) => typeof t === "number" && Number.isFinite(t)); @@ -151,8 +163,7 @@ function checked(capability: AudioApiCapability, body: AudioApiResponse): void { } const uploadedAsset = perSignal(async (analyzed: AudioSignal) => { - const { url, key, timeoutMs } = config(); - if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`); + const { url, headers, timeoutMs } = await granted(); const deadline = AbortSignal.timeout(timeoutMs); const boundedFetch: typeof fetch = (input, init) => fetch(input, { @@ -162,7 +173,7 @@ const uploadedAsset = perSignal(async (analyzed: AudioSignal) => { }); const client = new HyperframesCloudClient({ baseUrl: new URL(url).origin, - getAuthHeaders: () => ({ "x-api-key": key }), + getAuthHeaders: () => headers, fetchImpl: boundedFetch, }); const asset = await uploadAssetViaDirectUpload({ @@ -177,8 +188,7 @@ const uploadedAsset = perSignal(async (analyzed: AudioSignal) => { }); async function ask(analyzed: AudioSignal, capabilities: AudioApiCapability[]) { - const { url, key, timeoutMs } = config(); - if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`); + const { url, headers, timeoutMs } = await granted(); const request: AudioApiRequest = { version: 1, capabilities, @@ -188,7 +198,7 @@ async function ask(analyzed: AudioSignal, capabilities: AudioApiCapability[]) { method: "POST", redirect: "error", headers: { - "x-api-key": key, + ...headers, "idempotency-key": randomUUID(), "content-type": "application/json", }, @@ -258,16 +268,14 @@ function measured(capability: AudioApiCapability, signal: AudioSignal, body: Aud const API_CAPABILITIES: readonly AudioCapability[] = ["beats", "downbeats", "sections", "describe"]; -/** HeyGen's audio analysis service, or any service that speaks {@link AudioApiRequest}, configured by environment. */ +/** HeyGen's audio analysis service, or another that speaks {@link AudioApiRequest} when the environment names one. */ export const apiProvider: AudioAnalysisProvider = { id: "api", local: false, capabilities: API_CAPABILITIES, - available() { - const { url, key } = config(); - if (!url) return unavailable(`${URL_ENV} is not set`); - if (!key) return unavailable(`${KEY_ENV} is not set`); - return AVAILABLE; + async available() { + const found = await access(); + return typeof found === "string" ? unavailable(found) : AVAILABLE; }, async analyze(capability, signal) { const asked = capability as AudioApiCapability; diff --git a/packages/cli/src/audio/analysis/providers/shared.ts b/packages/cli/src/audio/analysis/providers/shared.ts index 5e78700bc19..2c0da5129b3 100644 --- a/packages/cli/src/audio/analysis/providers/shared.ts +++ b/packages/cli/src/audio/analysis/providers/shared.ts @@ -5,6 +5,9 @@ import { join } from "node:path"; import { promisify } from "node:util"; import type { AudioSignal, Availability, Downbeats } from "@hyperframes/core/audio-analysis"; import { medianInterval } from "@hyperframes/core/audio-analysis"; +import { buildAuthHeaders } from "../../../auth/client.js"; +import { tryResolveCredential } from "../../../auth/resolver.js"; +import { refreshIfNeeded, resolveCloudBaseUrl } from "../../../cloud/auth.js"; export const runFile = promisify(execFile); @@ -75,16 +78,49 @@ export function downbeatsWithMeter(beats: number[], downbeats: number[]): Downbe const round = (t: number) => Math.round(t * 1000) / 1000; export const rounded = (times: readonly number[]) => times.map(round); -export function serviceConfig( +export interface ServiceAccess { + url: string; + headers: Record; + timeoutMs: number; +} + +const NO_HEYGEN_CREDENTIAL = + "no HeyGen credential: set HEYGEN_API_KEY or run `hyperframes auth login --api-key`"; + +async function heygenHeaders(): Promise | string> { + try { + const credential = await tryResolveCredential(); + return credential ? buildAuthHeaders(await refreshIfNeeded(credential)) : NO_HEYGEN_CREDENTIAL; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } +} + +/** + * Where a HeyGen audio service is and how to call it, or why it cannot be called. By default that is `path` on the + * HeyGen API with the CLI's HeyGen credential. `env.url` points at another service, which then needs its own + * `env.key`, so the HeyGen credential never leaves for a host it was not issued for; `env.key` alone replaces the + * credential. + */ +export async function serviceAccess( env: { url: string; key: string; timeout: string }, + path: string, defaultMs: number, -) { +): Promise { const timeout = Number(process.env[env.timeout]); - return { - url: process.env[env.url]?.trim(), - key: process.env[env.key]?.trim(), - timeoutMs: timeout > 0 ? timeout : defaultMs, - }; + const timeoutMs = timeout > 0 ? timeout : defaultMs; + const url = process.env[env.url]?.trim(); + const key = process.env[env.key]?.trim(); + if (key) + return { + url: url || `${resolveCloudBaseUrl()}${path}`, + headers: { "x-api-key": key }, + timeoutMs, + }; + if (url) return `${env.url} is set but ${env.key} is not`; + const headers = await heygenHeaders(); + if (typeof headers === "string") return headers; + return { url: `${resolveCloudBaseUrl()}${path}`, headers, timeoutMs }; } function errorMessageIn(text: string): string | undefined { diff --git a/packages/cli/src/audio/read/read.test.ts b/packages/cli/src/audio/read/read.test.ts index 3728b8af93d..79bd94fe35e 100644 --- a/packages/cli/src/audio/read/read.test.ts +++ b/packages/cli/src/audio/read/read.test.ts @@ -3,7 +3,7 @@ import type { AddressInfo } from "node:net"; import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { readAudio } from "./read.js"; @@ -103,7 +103,14 @@ beforeAll(async () => { afterAll(() => new Promise((done) => server.close(() => done()))); +beforeEach(() => { + for (const name of ["HEYGEN_API_KEY", "HYPERFRAMES_API_KEY", "HEYGEN_ACCESS_TOKEN"]) + vi.stubEnv(name, ""); + vi.stubEnv("HEYGEN_API_URL", origin()); +}); + afterEach(() => { + vi.unstubAllEnvs(); delete process.env.HYPERFRAMES_AUDIO_STEMS_URL; delete process.env.HYPERFRAMES_AUDIO_API_KEY; }); @@ -130,7 +137,7 @@ describe.skipIf(!ffmpeg)("readAudio", () => { expect(first.analysis).toMatchObject({ separation: "mix", stems: [], music: null }); expect(first.analysis.voice!.sentences.map((s) => s.text)).toEqual(["一口气了解。", "伊朗"]); expect(first.analysis.voice!.medianPitchHz).toBeGreaterThan(170); - expect(first.analysis.warnings.join("\n")).toContain("HYPERFRAMES_AUDIO_STEMS_URL is not set"); + expect(first.analysis.warnings.join("\n")).toContain("no HeyGen credential"); const text = readFileSync(first.textPath, "utf8"); expect(text).toContain('voice "一口气了解。"'); expect(text).toContain("pause 1.0 s"); @@ -145,14 +152,13 @@ describe.skipIf(!ffmpeg)("readAudio", () => { expect(edited.analysis.voice!.sentences[0]!.text).toBe("换了"); }); - it("separates through the stems service, then reads effects from the drums and the bed from the rest", async () => { + it("separates through HeyGen's stems route with the CLI's HeyGen API key, then reads effects from the drums and the bed from the rest", async () => { stemFiles.vocals = wav("vocals.wav", 8, voiced); stemFiles.drums = wav("drums.wav", 8, boom); stemFiles.piano = wav("piano.wav", 8, bed); stemFiles.bass = wav("bass.wav", 8, () => 0); const path = wav("episode.wav", 8, (t) => voiced(t) + boom(t) + bed(t)); - process.env.HYPERFRAMES_AUDIO_STEMS_URL = `${origin()}/v3/audio/stems`; - process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key"; + vi.stubEnv("HEYGEN_API_KEY", "test-key"); const cacheDir = join(dir, "cache-stems"); const { analysis, textPath } = await readAudio(path, "episode.wav", { cacheDir }); expect(stemRequest).toEqual({ version: 1, audio: { type: "asset_id", asset_id: "episode" } }); diff --git a/packages/cli/src/audio/read/read.ts b/packages/cli/src/audio/read/read.ts index 7d317b732ec..730b93941ec 100644 --- a/packages/cli/src/audio/read/read.ts +++ b/packages/cli/src/audio/read/read.ts @@ -15,7 +15,7 @@ import { formatRead } from "./format.js"; import { readMusic } from "./music.js"; import { pitchTrack } from "./pitch.js"; import { PITCH_RATE, READ_RATE, decodeMono, mixed, round } from "./signal.js"; -import { DRUM_STEM, VOICE_STEM, cachedStems, fetchStems, stemsUnavailable } from "./stems.js"; +import { DRUM_STEM, VOICE_STEM, cachedStems, fetchStems, stemsAccess } from "./stems.js"; import { READ_VERSION, type ReadAudio } from "./types.js"; import { voiceFromActivity, voiceFromWords, type TimedWord } from "./voice.js"; @@ -77,13 +77,13 @@ async function stemsFor( warnings.push(`not separated into stems: quality ${quality} keeps audio on this machine`); return null; } - const missing = stemsUnavailable(); - if (missing) { - warnings.push(`not separated into stems: ${missing}`); + const access = await stemsAccess(); + if (typeof access === "string") { + warnings.push(`not separated into stems: ${access}`); return null; } try { - return await fetchStems(path, dir); + return await fetchStems(path, dir, access); } catch (error) { warnings.push(`not separated into stems: ${error instanceof Error ? error.message : error}`); return null; @@ -190,7 +190,7 @@ export async function readAudio( const stemsNowPossible = entry.analysis.separation === "mix" && quality === "best" && - (cachedStems(stemDir) !== null || stemsUnavailable() === null); + (cachedStems(stemDir) !== null || typeof (await stemsAccess()) !== "string"); if ( entry.analysis.version === READ_VERSION && entry.transcriptKey === transcriptKey && diff --git a/packages/cli/src/audio/read/stems.ts b/packages/cli/src/audio/read/stems.ts index 636dd0e7b0d..db3c619fe98 100644 --- a/packages/cli/src/audio/read/stems.ts +++ b/packages/cli/src/audio/read/stems.ts @@ -6,17 +6,23 @@ import { extname, join } from "node:path"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { HyperframesCloudClient } from "../../cloud/_gen/client.js"; import { uploadAssetViaDirectUpload } from "../../cloud/upload.js"; -import { runFile, serviceConfig, serviceFailure } from "../analysis/providers/shared.js"; +import { + runFile, + serviceAccess, + serviceFailure, + type ServiceAccess, +} from "../analysis/providers/shared.js"; /** * The stem separation service contract. * - * Request: `POST ` with `x-api-key: `, a new - * `Idempotency-Key`, and `{ version: 1, audio: { type: "asset_id", asset_id } }`. The recording is uploaded + * Request: `POST /v3/audio/stems` on the HeyGen API with the CLI's HeyGen credential, or `POST + * ` with `x-api-key: `; a new `Idempotency-Key`, and + * `{ version: 1, audio: { type: "asset_id", asset_id } }`. The recording is uploaded * first as 192 kb/s stereo MP3 through the same origin's asset direct-upload routes. * * Response: `{ data: StemsJob }`, or a bare `StemsJob`. While `status` is `pending` or `processing`, - * `GET /` returns the job again. A `completed` job lists one URL per stem; + * `GET /` returns the job again. A `completed` job lists one URL per stem; * the names are vocals, drums, bass, guitar, piano and other. */ interface StemsJob { @@ -36,16 +42,13 @@ const DONE = "stems.json"; export const VOICE_STEM = "vocals"; export const DRUM_STEM = "drums"; -const config = () => - serviceConfig({ url: URL_ENV, key: KEY_ENV, timeout: TIMEOUT_ENV }, DEFAULT_TIMEOUT_MS); - -/** Why stems cannot be fetched here, or null when they can. */ -export function stemsUnavailable(): string | null { - const { url, key } = config(); - if (!url) return `${URL_ENV} is not set`; - if (!key) return `${KEY_ENV} is not set`; - return null; -} +/** How to reach the stems service, or why it cannot be reached. */ +export const stemsAccess = () => + serviceAccess( + { url: URL_ENV, key: KEY_ENV, timeout: TIMEOUT_ENV }, + "/v3/audio/stems", + DEFAULT_TIMEOUT_MS, + ); /** Stems already fetched into `dir`, by name; null when there are none yet. */ export function cachedStems(dir: string): Record | null { @@ -79,11 +82,12 @@ async function encoded(path: string, dir: string): Promise { const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); /** Separate the recording through the configured service and save each stem into `dir`. */ -export async function fetchStems(path: string, dir: string): Promise> { - const { url, key, timeoutMs } = config(); - if (!url || !key) throw new Error(stemsUnavailable()!); +export async function fetchStems( + path: string, + dir: string, + { url, headers, timeoutMs }: ServiceAccess, +): Promise> { const deadline = Date.now() + timeoutMs; - const headers = { "x-api-key": key }; const scratch = await mkdtemp(join(tmpdir(), "hf-stems-")); try { const mp3 = await encoded(path, scratch); From 8f96c861c08d70b0567fbd246cdfc6e6ffa162b9 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:55:06 -0700 Subject: [PATCH 04/12] fix(cli): keep audio on the machine unless --quality best is asked for `hyperframes audio` defaulted to --quality best, so anyone with a HeyGen credential uploaded their audio, and could be billed, for an ordinary beats or --read run. tts, transcribe and remove-background never leave the machine unasked. The default is now local for both measuring and --read. best is unchanged and still uploads to /v3/audio/analysis and /v3/audio/stems. When a capability goes unanswered because a service was left out, or --read reads without stems, the warning says --quality best would add it. --- docs/packages/cli.mdx | 17 +++++--- .../cli/src/audio/analysis/analyze.test.ts | 28 +++++++++++++ packages/cli/src/audio/analysis/analyze.ts | 19 ++++++++- .../src/audio/analysis/providers/api.test.ts | 40 ++++++++++++------- packages/cli/src/audio/read/read.test.ts | 26 ++++++------ packages/cli/src/audio/read/read.ts | 6 ++- packages/cli/src/commands/audio.ts | 16 +++++--- 7 files changed, 111 insertions(+), 41 deletions(-) diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 1c707c94d2d..2f2935b67c6 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -610,7 +610,7 @@ Sentences come from a transcript: `--transcript `, or `.transcript.json` beside the recording. Without one, voice lines carry no words. -Effects and music are read from stems. At `--quality best` (the default) the +Effects and music are read from stems. At `--quality best` the recording is separated into vocals, drums, bass, guitar, piano and other by HeyGen's `/v3/audio/stems`, with your HeyGen credential (see [audio services](#audio-services)): it is uploaded as 192 kb/s MP3 through the @@ -663,21 +663,26 @@ answers: helper compiled once with the Xcode command line tools. A provider that cannot run, or fails, is passed over with a warning that says -why. `--quality fast|local|best` limits which providers may run (`local` keeps -the audio on this machine, `fast` allows only signal processing), -`--provider beats=signal` pins one, and `--full` shows each ladder step. +why. `--quality local|best|fast` limits which providers may run: `local`, the +default, keeps the audio on this machine; `best` also sends it to the audio +services below; `fast` allows only signal processing. When a capability goes +unanswered because a service was left out, a warning says `--quality best` could +answer it. `--provider beats=signal` pins a capability to one provider, and +`--full` shows each ladder step. Tune the analysis with `--onset-method`, `--threshold`, `--band lo,hi`, `--hop-ms`, `--onset-ratio`, `--tail-db` and `--silence-db`. Decoding needs FFmpeg. #### Audio services -Stems and the `api` provider call HeyGen with the same credential as the other +Only at `--quality best` is audio uploaded. Stems and the `api` provider then +call HeyGen, which may bill your account, with the same credential as the other HeyGen commands: `HEYGEN_API_KEY`, `HYPERFRAMES_API_KEY`, or one saved by `hyperframes auth login --api-key`, on the API at `HEYGEN_API_URL` when set. These routes take an API key; a `hyperframes auth login` browser sign-in is refused, and the analysis falls back with a warning. Without a credential, -stems are skipped and `api` is passed over. +stems are skipped and `api` is passed over. At `local`, the default, no credential +is looked up and nothing leaves the machine. To use another service, set `HYPERFRAMES_AUDIO_STEMS_URL` or `HYPERFRAMES_AUDIO_API_URL` to its full URL together with diff --git a/packages/cli/src/audio/analysis/analyze.test.ts b/packages/cli/src/audio/analysis/analyze.test.ts index 68e54b78738..3bb020c306a 100644 --- a/packages/cli/src/audio/analysis/analyze.test.ts +++ b/packages/cli/src/audio/analysis/analyze.test.ts @@ -83,6 +83,34 @@ describe("analyzeSignal", () => { }); }); +describe("analyzeSignal quality", () => { + const service: AudioAnalysisProvider = { + id: "service", + local: false, + capabilities: ["describe"], + analyze: async () => ({ tags: [{ label: "speech", score: 0.9 }] }), + }; + + it("keeps audio on this machine unless best is asked for, and says what best would add", async () => { + const local = await analyzeSignal(tone, { capabilities: ["describe"] }, [service]); + expect(local.values.describe).toBeNull(); + expect(local.ladder.describe).toEqual([ + { + provider: "service", + outcome: "excluded", + reason: "quality local keeps audio on this machine", + }, + ]); + expect(local.warnings.at(-1)).toContain("--quality best"); + + const best = await analyzeSignal(tone, { capabilities: ["describe"], quality: "best" }, [ + service, + ]); + expect(best.provenance.describe).toBe("service"); + expect(best.warnings).toEqual([]); + }); +}); + describe("envelopeHopMs", () => { it("keeps short sounds at 5 ms and coarsens long ones to about 2,000 values", () => { expect(envelopeHopMs(1)).toBe(5); diff --git a/packages/cli/src/audio/analysis/analyze.ts b/packages/cli/src/audio/analysis/analyze.ts index bc9850bd6c0..a30f0770a2d 100644 --- a/packages/cli/src/audio/analysis/analyze.ts +++ b/packages/cli/src/audio/analysis/analyze.ts @@ -36,6 +36,7 @@ function audioAnalysisProviders(): AudioAnalysisProvider[] { export interface AnalyzeOptions { capabilities: readonly AudioCapability[]; + /** Defaults to local: audio leaves the machine only when best is asked for. */ quality?: AnalysisQuality; /** Pin a capability to one provider, skipping the ladder. */ provider?: Partial>; @@ -103,6 +104,7 @@ export async function analyzeSignal( request: AnalyzeOptions, providers: readonly AudioAnalysisProvider[] = audioAnalysisProviders(), ): Promise { + const quality = request.quality ?? "local"; const { beats, bpm } = request.provider ?? {}; if (beats && bpm && beats !== bpm) throw new Error("beats and bpm must use the same provider"); const beatProvider = beats ?? bpm; @@ -120,7 +122,7 @@ export async function analyzeSignal( if (!result) { const withContext = capability === "onsets" && framing.cut; result = runLadder(capability, withContext ? signal : framing.windowed, providers, { - quality: request.quality, + quality, provider: capability === "beats" ? beatProvider : request.provider?.[capability], options: optionsFor(capability, request, analysis.analyzed), }); @@ -140,9 +142,24 @@ export async function analyzeSignal( const said = new Set(analysis.warnings); analysis.warnings.push(...result.warnings.filter((w) => !said.has(w))); } + analysis.warnings.push(...serviceHint(quality, analysis, request.capabilities)); return analysis; } +/** A warning when local left a capability unanswered that a service could have answered. */ +function serviceHint( + quality: AnalysisQuality, + analysis: Analysis, + capabilities: readonly AudioCapability[], +): string[] { + const leftToService = (capability: AudioCapability) => + analysis.values[capability] == null && + (analysis.ladder[capability] ?? []).some((s) => s.outcome === "excluded"); + return quality === "local" && capabilities.some(leftToService) + ? ["a service could have answered what is missing: --quality best uploads the audio to HeyGen"] + : []; +} + /** Tempo is read off the beats of whichever provider placed them, so the two never disagree. */ function tempoFromBeats(beats: LadderResult): LadderResult { const times = (beats.value as Beats | null)?.times; diff --git a/packages/cli/src/audio/analysis/providers/api.test.ts b/packages/cli/src/audio/analysis/providers/api.test.ts index 2444f9300e0..885d334a1ed 100644 --- a/packages/cli/src/audio/analysis/providers/api.test.ts +++ b/packages/cli/src/audio/analysis/providers/api.test.ts @@ -204,10 +204,11 @@ describe("apiProvider", () => { downbeats: { times: measured.filter((_, k) => k % 4 === 0).map((t) => t - 0.02), meter: 4 }, } satisfies AudioApiResponse, }); - const analysis = await analyzeSignal(signal, { capabilities: ["beats", "downbeats", "bpm"] }, [ - apiProvider, - signalProvider, - ]); + const analysis = await analyzeSignal( + signal, + { quality: "best", capabilities: ["beats", "downbeats", "bpm"] }, + [apiProvider, signalProvider], + ); expect(received).toHaveLength(1); const [call] = received; @@ -249,6 +250,7 @@ describe("apiProvider", () => { configure(); reply = () => ({ status: 200, body: { version: 1, beats: { times: [0.5, 1, 1.5] } } }); const analysis = await analyzeSignal(clicks(8), { + quality: "best", capabilities, provider: { bpm: "signal" }, }); @@ -271,6 +273,7 @@ describe("apiProvider", () => { }); await expect( analyzeSignal(clicks(8), { + quality: "best", capabilities: ["describe", "beats", "bpm"], provider, }), @@ -296,10 +299,11 @@ describe("apiProvider", () => { describe: { text: "clicks", tags: [{ label: "click", score: 0.9 }] }, }, }); - const analysis = await analyzeSignal(signal, { capabilities: ["sections", "describe"] }, [ - apiProvider, - signalProvider, - ]); + const analysis = await analyzeSignal( + signal, + { quality: "best", capabilities: ["sections", "describe"] }, + [apiProvider, signalProvider], + ); expect(received.map((r) => r.request.capabilities)).toEqual([["sections"], ["describe"]]); expect(new Set(received.map((r) => r.idempotencyKey)).size).toBe(2); expect(uploads).toBe(1); @@ -320,7 +324,7 @@ describe("apiProvider", () => { it("falls through to signal processing with a warning when the service fails or answers in another shape", async () => { configure(); reply = () => ({ status: 503, body: { error: { message: "busy" } } }); - const failed = await analyzeSignal(clicks(8), { capabilities: ["beats"] }, [ + const failed = await analyzeSignal(clicks(8), { quality: "best", capabilities: ["beats"] }, [ apiProvider, signalProvider, ]); @@ -330,7 +334,7 @@ describe("apiProvider", () => { ]); reply = () => ({ status: 200, body: { version: 1, beats: { times: ["soon"] } } }); - const garbled = await analyzeSignal(clicks(8), { capabilities: ["beats"] }, [ + const garbled = await analyzeSignal(clicks(8), { quality: "best", capabilities: ["beats"] }, [ apiProvider, signalProvider, ]); @@ -352,7 +356,7 @@ describe("apiProvider", () => { reply = () => ({ status: 200, body: { version: 1, describe: { tags: [] } } }); await analyzeSignal( { sampleRate: rate, channels: [left, right] }, - { capabilities: ["describe"] }, + { quality: "best", capabilities: ["describe"] }, [apiProvider], ); @@ -402,7 +406,7 @@ describe("apiProvider", () => { reply = () => ({ status: 200, body: { version: 1, describe: { tags: [] } } }); await analyzeSignal( { sampleRate: rate, channels: [left, right] }, - { capabilities: ["describe"] }, + { quality: "best", capabilities: ["describe"] }, [apiProvider], ); const wav = Buffer.from(received[0]!.audio.bytes); @@ -417,14 +421,22 @@ describe("apiProvider", () => { configure(); enveloped = false; reply = () => ({ status: 200, body: { version: 1, describe: { text: "bare", tags: [] } } }); - const analysis = await analyzeSignal(clicks(2), { capabilities: ["describe"] }, [apiProvider]); + const analysis = await analyzeSignal( + clicks(2), + { quality: "best", capabilities: ["describe"] }, + [apiProvider], + ); expect(analysis.values.describe).toEqual({ text: "bare", tags: [] }); }); it("never sends the key anywhere but the x-api-key header", async () => { configure(); reply = () => ({ status: 401, body: { error: { message: "bad key" } } }); - const analysis = await analyzeSignal(clicks(8), { capabilities: ["describe"] }, [apiProvider]); + const analysis = await analyzeSignal( + clicks(8), + { quality: "best", capabilities: ["describe"] }, + [apiProvider], + ); expect(JSON.stringify(analysis)).not.toContain("test-key"); const [call] = received; expect(call).toMatchObject({ apiKey: "test-key", authorization: undefined }); diff --git a/packages/cli/src/audio/read/read.test.ts b/packages/cli/src/audio/read/read.test.ts index 79bd94fe35e..d9b54a7a494 100644 --- a/packages/cli/src/audio/read/read.test.ts +++ b/packages/cli/src/audio/read/read.test.ts @@ -132,7 +132,7 @@ describe.skipIf(!ffmpeg)("readAudio", () => { }), ); const cacheDir = join(dir, "cache-mix"); - const first = await readAudio(path, "talk.wav", { cacheDir }); + const first = await readAudio(path, "talk.wav", { cacheDir, quality: "best" }); expect(first.cached).toBe(false); expect(first.analysis).toMatchObject({ separation: "mix", stems: [], music: null }); expect(first.analysis.voice!.sentences.map((s) => s.text)).toEqual(["一口气了解。", "伊朗"]); @@ -141,13 +141,13 @@ describe.skipIf(!ffmpeg)("readAudio", () => { const text = readFileSync(first.textPath, "utf8"); expect(text).toContain('voice "一口气了解。"'); expect(text).toContain("pause 1.0 s"); - const again = await readAudio(path, "talk.wav", { cacheDir }); + const again = await readAudio(path, "talk.wav", { cacheDir, quality: "best" }); expect(again.cached).toBe(true); writeFileSync( `${path}.transcript.json`, JSON.stringify({ words: [{ text: "换了", start: 0.5, end: 2 }] }), ); - const edited = await readAudio(path, "talk.wav", { cacheDir }); + const edited = await readAudio(path, "talk.wav", { cacheDir, quality: "best" }); expect(edited.cached).toBe(false); expect(edited.analysis.voice!.sentences[0]!.text).toBe("换了"); }); @@ -160,7 +160,10 @@ describe.skipIf(!ffmpeg)("readAudio", () => { const path = wav("episode.wav", 8, (t) => voiced(t) + boom(t) + bed(t)); vi.stubEnv("HEYGEN_API_KEY", "test-key"); const cacheDir = join(dir, "cache-stems"); - const { analysis, textPath } = await readAudio(path, "episode.wav", { cacheDir }); + const { analysis, textPath } = await readAudio(path, "episode.wav", { + cacheDir, + quality: "best", + }); expect(stemRequest).toEqual({ version: 1, audio: { type: "asset_id", asset_id: "episode" } }); expect(polls).toBe(1); expect(analysis).toMatchObject({ @@ -177,20 +180,19 @@ describe.skipIf(!ffmpeg)("readAudio", () => { expect(analysis.music!.segments.map((s) => s.entry)).toEqual(["start", "resumes"]); expect(readFileSync(textPath, "utf8")).toContain("effect boom (S1): low"); expect(existsSync(join(cacheDir, "stems"))).toBe(true); - await readAudio(path, "episode.wav", { cacheDir }); + await readAudio(path, "episode.wav", { cacheDir, quality: "best" }); expect(stemCalls).toBe(1); }); - it("keeps audio on the machine at quality local, and reads neither voice nor effects from a bare mix", async () => { - process.env.HYPERFRAMES_AUDIO_STEMS_URL = `${origin()}/v3/audio/stems`; - process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key"; + it("keeps audio on the machine by default, and reads neither voice nor effects from a bare mix", async () => { const path = wav("local.wav", 3, voiced); - const { analysis } = await readAudio(path, "local.wav", { - cacheDir: join(dir, "cache-local"), - quality: "local", - }); + vi.stubEnv("HEYGEN_API_KEY", "test-key"); + const callsBefore = stemCalls; + const { analysis } = await readAudio(path, "local.wav", { cacheDir: join(dir, "cache-local") }); expect(analysis).toMatchObject({ separation: "mix", voice: null, effects: null }); expect(analysis.warnings[0]).toContain("quality local keeps audio on this machine"); + expect(analysis.warnings[0]).toContain("--quality best"); + expect(stemCalls).toBe(callsBefore); expect(analysis.warnings[1]).toContain("no transcript and no stems"); }); }); diff --git a/packages/cli/src/audio/read/read.ts b/packages/cli/src/audio/read/read.ts index 730b93941ec..7321f3ccff8 100644 --- a/packages/cli/src/audio/read/read.ts +++ b/packages/cli/src/audio/read/read.ts @@ -74,7 +74,9 @@ async function stemsFor( const cached = cachedStems(dir); if (cached) return cached; if (quality !== "best") { - warnings.push(`not separated into stems: quality ${quality} keeps audio on this machine`); + warnings.push( + `not separated into stems: quality ${quality} keeps audio on this machine; --quality best separates it through HeyGen`, + ); return null; } const access = await stemsAccess(); @@ -176,7 +178,7 @@ export async function readAudio( file: string, options: ReadOptions, ): Promise { - const quality = options.quality ?? "best"; + const quality = options.quality ?? "local"; const sha = await sha256(path); const transcript = transcriptFor(path, options.transcript); const transcriptKey = fingerprint(transcript); diff --git a/packages/cli/src/commands/audio.ts b/packages/cli/src/commands/audio.ts index 9541eda8538..609e72305b0 100644 --- a/packages/cli/src/commands/audio.ts +++ b/packages/cli/src/commands/audio.ts @@ -125,9 +125,14 @@ interface AudioFlags { json?: boolean; } +function parseQuality(flags: AudioFlags): AnalysisQuality { + const quality = (flags.quality ?? "local") as AnalysisQuality; + if (!QUALITIES.has(quality)) throw new CliUsageError("--quality is local, best or fast"); + return quality; +} + function analyzeRequest(flags: AudioFlags): AnalyzeOptions { - const quality = (flags.quality ?? "best") as AnalysisQuality; - if (!QUALITIES.has(quality)) throw new CliUsageError("--quality is best, local or fast"); + const quality = parseQuality(flags); const method = flags["onset-method"] as OnsetMethod | undefined; if (method && method !== "spectral-flux" && method !== "energy") throw new CliUsageError("--onset-method is spectral-flux or energy"); @@ -173,9 +178,7 @@ function readQuality(files: string[], flags: AudioFlags): AnalysisQuality { if (files.length !== 1) throw new CliUsageError("--read reads exactly one file"); if (flags.element || flags.at || flags["place-at"]) throw new CliUsageError("--read takes a file and --range, not a film query"); - const quality = (flags.quality ?? "best") as AnalysisQuality; - if (!QUALITIES.has(quality)) throw new CliUsageError("--quality is best, local or fast"); - return quality; + return parseQuality(flags); } async function read(files: string[], flags: AudioFlags) { @@ -221,7 +224,8 @@ export default defineCommand({ }, quality: { type: "string", - description: "best (any provider), local (this machine), fast (signal only)", + description: + "local (this machine, the default), best (also uploads to HeyGen's audio services, which may bill your account), fast (signal only)", }, provider: { type: "string", description: "Pin capabilities to providers: onsets=signal,..." }, "onset-method": { type: "string", description: "spectral-flux (default) or energy" }, From 28f3997aa54ec092d8f09ce3f48696273d1d07f5 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:56:42 -0700 Subject: [PATCH 05/12] fix(cli): bound stem transfers and preserve recording timestamps --- docs/packages/cli.mdx | 11 +-- .../cli/src/audio/analysis/providers/api.ts | 15 +--- .../src/audio/analysis/providers/shared.ts | 7 +- packages/cli/src/audio/read/effects.test.ts | 2 +- packages/cli/src/audio/read/read.test.ts | 87 +++++++++++++------ packages/cli/src/audio/read/signal.test.ts | 13 +++ packages/cli/src/audio/read/signal.ts | 4 +- packages/cli/src/audio/read/stems.ts | 37 ++++---- packages/cli/src/audio/read/types.ts | 2 +- packages/cli/src/cloud/upload.ts | 8 +- 10 files changed, 113 insertions(+), 73 deletions(-) create mode 100644 packages/cli/src/audio/read/signal.test.ts diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 1c707c94d2d..cc22781dc8f 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -673,11 +673,12 @@ FFmpeg. #### Audio services Stems and the `api` provider call HeyGen with the same credential as the other -HeyGen commands: `HEYGEN_API_KEY`, `HYPERFRAMES_API_KEY`, or one saved by -`hyperframes auth login --api-key`, on the API at `HEYGEN_API_URL` when set. -These routes take an API key; a `hyperframes auth login` browser sign-in is -refused, and the analysis falls back with a warning. Without a credential, -stems are skipped and `api` is passed over. +HeyGen commands: `HEYGEN_API_KEY`, `HYPERFRAMES_API_KEY`, `HEYGEN_ACCESS_TOKEN`, +or a credential saved by `hyperframes auth login`, on the API at +`HEYGEN_API_URL` when set. API keys are sent as `x-api-key`; OAuth credentials +are sent as bearer tokens. If the service refuses a request, analysis falls +back with a warning. Without a credential, stems are skipped and `api` is +passed over. To use another service, set `HYPERFRAMES_AUDIO_STEMS_URL` or `HYPERFRAMES_AUDIO_API_URL` to its full URL together with diff --git a/packages/cli/src/audio/analysis/providers/api.ts b/packages/cli/src/audio/analysis/providers/api.ts index 656f123b505..5cc1ff7492c 100644 --- a/packages/cli/src/audio/analysis/providers/api.ts +++ b/packages/cli/src/audio/analysis/providers/api.ts @@ -22,17 +22,10 @@ import { } from "./shared.js"; /** - * The audio analysis service contract. - * - * Request: `POST /v3/audio/analysis` on the HeyGen API with the CLI's HeyGen credential, or `POST - * ` exactly as configured with `x-api-key: `; a new - * `Idempotency-Key` per request, and an {@link AudioApiRequest} JSON body. - * The analyzed span is uploaded once as 16-bit PCM WAV, mono at 16 kHz, using the same origin - * public asset direct-upload and completion routes. The body references that finalized asset. - * - * Response: `200` with `application/json`, `{ "data": AudioApiResponse }`; a bare {@link AudioApiResponse} is read - * too. Every time is in seconds from the start of the uploaded audio. A capability the service cannot answer is left - * out of the response. Any other status is a failure; its body may be `{ "error": { "message": string } }`. + * POST an asset-backed request to the configured audio service with a fresh Idempotency-Key. + * Uploads are 16 kHz mono WAV through the same origin's direct-upload routes. + * Responses may be bare or wrapped in `data`; times start at the uploaded audio's beginning. + * Unavailable capabilities are omitted. Other HTTP statuses fail with the service's error. */ export interface AudioApiRequest { version: 1; diff --git a/packages/cli/src/audio/analysis/providers/shared.ts b/packages/cli/src/audio/analysis/providers/shared.ts index 2c0da5129b3..2181b0f1f27 100644 --- a/packages/cli/src/audio/analysis/providers/shared.ts +++ b/packages/cli/src/audio/analysis/providers/shared.ts @@ -96,12 +96,7 @@ async function heygenHeaders(): Promise | string> { } } -/** - * Where a HeyGen audio service is and how to call it, or why it cannot be called. By default that is `path` on the - * HeyGen API with the CLI's HeyGen credential. `env.url` points at another service, which then needs its own - * `env.key`, so the HeyGen credential never leaves for a host it was not issued for; `env.key` alone replaces the - * credential. - */ +/** An explicit service URL requires its own key; never forward the HeyGen credential to it. */ export async function serviceAccess( env: { url: string; key: string; timeout: string }, path: string, diff --git a/packages/cli/src/audio/read/effects.test.ts b/packages/cli/src/audio/read/effects.test.ts index 2d6005b841e..ee8cbc96555 100644 --- a/packages/cli/src/audio/read/effects.test.ts +++ b/packages/cli/src/audio/read/effects.test.ts @@ -42,7 +42,7 @@ describe("findEffects", () => { expect(events.map((e) => [Math.round(e.t * 10) / 10, e.kind])).toEqual([ [1, "boom"], [3.2, "tick"], - [5, "riser"], + [5.1, "riser"], ]); expect(events[0]).toMatchObject({ band: "low", count: 1, overMusicDb: null }); expect(events[0]!.tail).toBeGreaterThan(0.6); diff --git a/packages/cli/src/audio/read/read.test.ts b/packages/cli/src/audio/read/read.test.ts index 79bd94fe35e..57421a83f83 100644 --- a/packages/cli/src/audio/read/read.test.ts +++ b/packages/cli/src/audio/read/read.test.ts @@ -6,6 +6,7 @@ import { join } from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { readAudio } from "./read.js"; +import { fetchStems } from "./stems.js"; const RATE = 22_050; const dir = mkdtempSync(join(tmpdir(), "hf-audio-read-")); @@ -44,6 +45,8 @@ const bed = (t: number) => : 0.08 * Math.sin(2 * Math.PI * 261.63 * t) + 0.08 * Math.sin(2 * Math.PI * 392 * t); let server: Server; +let stalledPath: string | null = null; +let onStalled = () => {}; let polls = 0; let stemCalls = 0; let stemRequest: unknown = null; @@ -59,40 +62,46 @@ const origin = () => `http://127.0.0.1:${(server.address() as AddressInfo).port} type Reply = (status: number, value: unknown) => void; -const routes: Record void> = { - "POST /v3/assets/direct-uploads": (reply) => - reply(200, { - data: { asset_id: "episode", upload_url: `${origin()}/put`, upload_headers: {} }, - }), - "PUT /put": (reply) => reply(200, {}), - "POST /v3/assets/episode/complete": (reply) => reply(200, { data: { asset_id: "episode" } }), - "POST /v3/audio/stems": (reply, req, bytes) => { - expect(req.headers["x-api-key"]).toBe("test-key"); - stemCalls++; - stemRequest = JSON.parse(bytes.toString()); - reply(202, { data: { id: "job-1", status: "processing" } }); - }, - "GET /v3/audio/stems/job-1": (reply) => { - polls++; - reply(200, { - data: { - id: "job-1", - status: "completed", - stems: Object.keys(stemFiles).map((name) => ({ - name, - url: `${origin()}/stem/${name}.wav`, - })), - }, - }); - }, -}; +const routes = new Map void>( + Object.entries({ + "POST /v3/assets/direct-uploads": (reply) => + reply(200, { + data: { asset_id: "episode", upload_url: `${origin()}/put`, upload_headers: {} }, + }), + "PUT /put": (reply) => reply(200, {}), + "POST /v3/assets/episode/complete": (reply) => reply(200, { data: { asset_id: "episode" } }), + "POST /v3/audio/stems": (reply, req, bytes) => { + expect(req.headers["x-api-key"]).toBe("test-key"); + stemCalls++; + stemRequest = JSON.parse(bytes.toString()); + reply(202, { data: { id: "job-1", status: "processing" } }); + }, + "GET /v3/audio/stems/job-1": (reply) => { + polls++; + reply(200, { + data: { + id: "job-1", + status: "completed", + stems: Object.keys(stemFiles).map((name) => ({ + name, + url: `${origin()}/stem/${name}.wav`, + })), + }, + }); + }, + }), +); beforeAll(async () => { server = createServer(async (req, res) => { + if (req.url === stalledPath) { + onStalled(); + return; + } const bytes = await body(req); const reply: Reply = (status, value) => res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(value)); - const route = routes[`${req.method} ${req.url}`]; + const route = routes.get(`${req.method} ${req.url}`); if (route) return route(reply, req, bytes); const stem = req.url?.match(/^\/stem\/(\w+)\.wav$/)?.[1]; if (stem && stemFiles[stem]) return res.writeHead(200).end(readFileSync(stemFiles[stem])); @@ -110,6 +119,8 @@ beforeEach(() => { }); afterEach(() => { + stalledPath = null; + vi.restoreAllMocks(); vi.unstubAllEnvs(); delete process.env.HYPERFRAMES_AUDIO_STEMS_URL; delete process.env.HYPERFRAMES_AUDIO_API_KEY; @@ -157,6 +168,7 @@ describe.skipIf(!ffmpeg)("readAudio", () => { stemFiles.drums = wav("drums.wav", 8, boom); stemFiles.piano = wav("piano.wav", 8, bed); stemFiles.bass = wav("bass.wav", 8, () => 0); + stemFiles["../../escape"] = stemFiles.bass; const path = wav("episode.wav", 8, (t) => voiced(t) + boom(t) + bed(t)); vi.stubEnv("HEYGEN_API_KEY", "test-key"); const cacheDir = join(dir, "cache-stems"); @@ -181,6 +193,25 @@ describe.skipIf(!ffmpeg)("readAudio", () => { expect(stemCalls).toBe(1); }); + it.each(["/put", "/v3/audio/stems"])( + "aborts a stalled %s request at the separation deadline", + async (endpoint) => { + const controller = new AbortController(); + const timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal); + stalledPath = endpoint; + onStalled = () => controller.abort(); + const path = wav("timeout.wav", 1, voiced); + await expect( + fetchStems(path, join(dir, `timeout-${endpoint.replaceAll("/", "-")}`), { + url: `${origin()}/v3/audio/stems`, + headers: { "x-api-key": "test-key" }, + timeoutMs: 1000, + }), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(timeout).toHaveBeenCalledWith(1000); + }, + ); + it("keeps audio on the machine at quality local, and reads neither voice nor effects from a bare mix", async () => { process.env.HYPERFRAMES_AUDIO_STEMS_URL = `${origin()}/v3/audio/stems`; process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key"; diff --git a/packages/cli/src/audio/read/signal.test.ts b/packages/cli/src/audio/read/signal.test.ts new file mode 100644 index 00000000000..cc882316707 --- /dev/null +++ b/packages/cli/src/audio/read/signal.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { levels, READ_RATE } from "./signal.js"; + +describe("levels", () => { + it("keeps fractional sample hops aligned late in a recording", () => { + const samples = new Float32Array(601 * READ_RATE); + samples.fill(0.5, 600 * READ_RATE, Math.round(600.1 * READ_RATE)); + const db = levels(samples, READ_RATE, 0.01); + expect(db).toHaveLength(60100); + expect(db[60005]).toBeCloseTo(20 * Math.log10(0.5), 3); + expect(db[59990]).toBe(-120); + }); +}); diff --git a/packages/cli/src/audio/read/signal.ts b/packages/cli/src/audio/read/signal.ts index 837b1c49f46..cb776fed980 100644 --- a/packages/cli/src/audio/read/signal.ts +++ b/packages/cli/src/audio/read/signal.ts @@ -31,14 +31,14 @@ export function levels( hop: number, window = hop * 2, ): Float32Array { - const hopSamples = Math.max(1, Math.round(hop * sampleRate)); + const hopSamples = Math.max(1, hop * sampleRate); const half = Math.max(1, Math.round((window * sampleRate) / 2)); const squares = new Float64Array(samples.length + 1); for (let i = 0; i < samples.length; i++) squares[i + 1] = squares[i]! + samples[i]! ** 2; const frames = Math.ceil(samples.length / hopSamples); const out = new Float32Array(frames); for (let f = 0; f < frames; f++) { - const center = f * hopSamples; + const center = Math.round(f * hopSamples); const a = Math.max(0, center - half); const b = Math.min(samples.length, center + half); const power = b > a ? (squares[b]! - squares[a]!) / (b - a) : 0; diff --git a/packages/cli/src/audio/read/stems.ts b/packages/cli/src/audio/read/stems.ts index db3c619fe98..65613469d78 100644 --- a/packages/cli/src/audio/read/stems.ts +++ b/packages/cli/src/audio/read/stems.ts @@ -2,8 +2,10 @@ import { randomUUID } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { extname, join } from "node:path"; +import { join } from "node:path"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { setTimeout as sleep } from "node:timers/promises"; +import { downloadToFile } from "../../cloud/download.js"; import { HyperframesCloudClient } from "../../cloud/_gen/client.js"; import { uploadAssetViaDirectUpload } from "../../cloud/upload.js"; import { @@ -38,6 +40,7 @@ const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS"; const DEFAULT_TIMEOUT_MS = 30 * 60_000; const POLL_MS = 5000; const DONE = "stems.json"; +const STEM_NAMES = ["vocals", "drums", "bass", "guitar", "piano", "other"] as const; export const VOICE_STEM = "vocals"; export const DRUM_STEM = "drums"; @@ -67,34 +70,32 @@ async function job(response: Response): Promise { return value; } -async function encoded(path: string, dir: string): Promise { +async function encoded(path: string, dir: string, signal: AbortSignal): Promise { const ffmpeg = findFfBinary("ffmpeg", { configuredMustExist: true }); if (!ffmpeg) throw new Error("ffmpeg is required to separate stems; run: npx hyperframes doctor"); const out = join(dir, "upload.mp3"); await runFile( ffmpeg, ["-v", "error", "-y", "-i", path, "-vn", "-ac", "2", "-ar", "44100", "-b:a", "192k", out], - { windowsHide: true, maxBuffer: 8 * 1024 * 1024 }, + { windowsHide: true, maxBuffer: 8 * 1024 * 1024, signal }, ); return out; } -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - /** Separate the recording through the configured service and save each stem into `dir`. */ export async function fetchStems( path: string, dir: string, { url, headers, timeoutMs }: ServiceAccess, ): Promise> { - const deadline = Date.now() + timeoutMs; + const signal = AbortSignal.timeout(timeoutMs); const scratch = await mkdtemp(join(tmpdir(), "hf-stems-")); try { - const mp3 = await encoded(path, scratch); + const mp3 = await encoded(path, scratch, signal); const client = new HyperframesCloudClient({ baseUrl: new URL(url).origin, getAuthHeaders: () => headers, - fetchImpl: (input, init) => fetch(input, { ...init, redirect: "error" }), + fetchImpl: (input, init) => fetch(input, { ...init, redirect: "error", signal }), }); const asset = await uploadAssetViaDirectUpload({ client, @@ -103,10 +104,12 @@ export async function fetchStems( filename: "audio.mp3", contentType: "audio/mpeg", idempotencyKey: randomUUID(), + signal, }); let current = await job( await fetch(url, { method: "POST", + signal, redirect: "error", headers: { ...headers, @@ -117,13 +120,12 @@ export async function fetchStems( }), ); while (current.status === "pending" || current.status === "processing") { - if (Date.now() > deadline) - throw new Error(`stem separation did not finish within ${Math.round(timeoutMs / 1000)} s`); - await sleep(POLL_MS); + await sleep(POLL_MS, undefined, { signal }); current = await job( await fetch(`${url.replace(/\/$/, "")}/${encodeURIComponent(current.id)}`, { redirect: "error", headers, + signal, }), ); } @@ -136,13 +138,12 @@ export async function fetchStems( throw new Error("the stems service returned no vocals stem"); mkdirSync(dir, { recursive: true }); const names: Record = {}; - for (const stem of stems) { - const response = await fetch(stem.url); - if (!response.ok) - throw new Error(`could not download the ${stem.name} stem: ${response.status}`); - const file = `${stem.name.replace(/[^\w-]/g, "_")}${extname(new URL(stem.url).pathname) || ".mp3"}`; - writeFileSync(join(dir, file), Buffer.from(await response.arrayBuffer())); - names[stem.name] = file; + for (const name of STEM_NAMES) { + const stem = stems.find((candidate) => candidate.name === name); + if (!stem) continue; + const file = `${name}.audio`; + await downloadToFile(stem.url, join(dir, file), { signal }); + names[name] = file; } writeFileSync(join(dir, DONE), JSON.stringify(names)); return cachedStems(dir)!; diff --git a/packages/cli/src/audio/read/types.ts b/packages/cli/src/audio/read/types.ts index 608a2db5002..ad8c877019d 100644 --- a/packages/cli/src/audio/read/types.ts +++ b/packages/cli/src/audio/read/types.ts @@ -1,6 +1,6 @@ import type { Span } from "@hyperframes/core/audio-analysis"; -export const READ_VERSION = 1; +export const READ_VERSION = 2; export type Separation = "stems" | "mix"; diff --git a/packages/cli/src/cloud/upload.ts b/packages/cli/src/cloud/upload.ts index 2598ab5305f..419a9112995 100644 --- a/packages/cli/src/cloud/upload.ts +++ b/packages/cli/src/cloud/upload.ts @@ -56,6 +56,7 @@ export interface UploadZipViaDirectOptions { idempotencyKey?: string; fetchImpl?: typeof fetch; onProgress?: (event: UploadProgressEvent) => void; + signal?: AbortSignal; } // Per api-docs: completeAssetUpload can return 409 "Uploaded object not found @@ -103,17 +104,19 @@ async function putBytesToPresignedUrl( uploadHeaders: Record, body: Uint8Array | UploadFileSource, contentType: string, + signal?: AbortSignal, ): Promise { const headers: Record = { "content-type": contentType, ...normalizeUploadHeaders(uploadHeaders), }; - if (!(body instanceof Uint8Array)) return putFileToPresignedUrl(uploadUrl, headers, body); + if (!(body instanceof Uint8Array)) return putFileToPresignedUrl(uploadUrl, headers, body, signal); // `Uint8Array` is a valid `BodyInit` at runtime but // not strictly assignable per lib.dom.d.ts — cast rather than copy, // since a 200MB buffer copy would be wasteful. const res = await fetchImpl(uploadUrl, { method: "PUT", + signal, headers, body: body as unknown as BodyInit, }); @@ -132,10 +135,12 @@ async function putFileToPresignedUrl( uploadUrl: string, headers: Record, file: UploadFileSource, + signal?: AbortSignal, ): Promise { const send = new URL(uploadUrl).protocol === "https:" ? httpsRequest : httpRequest; const req = send(uploadUrl, { method: "PUT", + signal, headers: { ...headers, "content-length": String(file.size) }, }); const response = new Promise((resolve, reject) => { @@ -221,6 +226,7 @@ export async function uploadAssetViaDirectUpload( initialize.upload_headers, body, opts.contentType, + opts.signal, ); opts.onProgress?.({ phase: "upload", percent: 100 }); From 0804697e7bfbf508bc4aeca69769a7fa29fdaf8e Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:57:05 -0700 Subject: [PATCH 06/12] fix(cli): an empty audio service URL keeps the service off HyperFrames Desktop blanks HYPERFRAMES_AUDIO_API_URL and its key while the analysis route is behind a flag, but gives runs HEYGEN_API_KEY and HEYGEN_API_URL for its gateway. Falling back to that credential would let a run that passes --quality best reach the route anyway. An empty URL now turns the service off, whatever credential is present. --- docs/packages/cli.mdx | 3 ++- packages/cli/src/audio/analysis/providers/api.test.ts | 10 ++++++++++ packages/cli/src/audio/analysis/providers/shared.ts | 4 +++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 2f2935b67c6..124d74bc011 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -687,7 +687,8 @@ is looked up and nothing leaves the machine. To use another service, set `HYPERFRAMES_AUDIO_STEMS_URL` or `HYPERFRAMES_AUDIO_API_URL` to its full URL together with `HYPERFRAMES_AUDIO_API_KEY`, which is sent as `x-api-key`. Your HeyGen -credential is never sent to a URL set this way. +credential is never sent to a URL set this way. Setting either URL to an empty +string turns that service off, whatever credential is present. ## Look at it diff --git a/packages/cli/src/audio/analysis/providers/api.test.ts b/packages/cli/src/audio/analysis/providers/api.test.ts index 885d334a1ed..7dc5e0cdc3f 100644 --- a/packages/cli/src/audio/analysis/providers/api.test.ts +++ b/packages/cli/src/audio/analysis/providers/api.test.ts @@ -149,6 +149,16 @@ describe("apiProvider", () => { expect(received.map((r) => [r.url, r.apiKey])).toEqual([["/v3/audio/analysis", "test-key"]]); }); + it("stays off when the host blanks HYPERFRAMES_AUDIO_API_URL, whatever HeyGen credential it holds", async () => { + vi.stubEnv("HEYGEN_API_KEY", "gateway-token"); + process.env.HYPERFRAMES_AUDIO_API_URL = ""; + process.env.HYPERFRAMES_AUDIO_API_KEY = ""; + expect(await apiProvider.available!()).toEqual({ + ok: false, + reason: "HYPERFRAMES_AUDIO_API_URL is empty, which turns this service off", + }); + }); + it("never sends the HeyGen credential to a service named by HYPERFRAMES_AUDIO_API_URL", async () => { vi.stubEnv("HEYGEN_API_KEY", "heygen-key"); process.env.HYPERFRAMES_AUDIO_API_URL = "http://127.0.0.1:1/"; diff --git a/packages/cli/src/audio/analysis/providers/shared.ts b/packages/cli/src/audio/analysis/providers/shared.ts index 2c0da5129b3..f5901912b84 100644 --- a/packages/cli/src/audio/analysis/providers/shared.ts +++ b/packages/cli/src/audio/analysis/providers/shared.ts @@ -100,7 +100,8 @@ async function heygenHeaders(): Promise | string> { * Where a HeyGen audio service is and how to call it, or why it cannot be called. By default that is `path` on the * HeyGen API with the CLI's HeyGen credential. `env.url` points at another service, which then needs its own * `env.key`, so the HeyGen credential never leaves for a host it was not issued for; `env.key` alone replaces the - * credential. + * credential. `env.url` set to an empty string turns the service off, so a host that runs the CLI can keep it local + * even though the CLI holds a HeyGen credential. */ export async function serviceAccess( env: { url: string; key: string; timeout: string }, @@ -111,6 +112,7 @@ export async function serviceAccess( const timeoutMs = timeout > 0 ? timeout : defaultMs; const url = process.env[env.url]?.trim(); const key = process.env[env.key]?.trim(); + if (url === "") return `${env.url} is empty, which turns this service off`; if (key) return { url: url || `${resolveCloudBaseUrl()}${path}`, From 8bba02aab2e7eb877ba007c48662b2a43c992c9e Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:25:05 -0700 Subject: [PATCH 07/12] fix(ci): capture package name in release-age exclusion (cherry picked from commit f9c6ba35739df899ffa00f5d744e4d65860d5cea) --- scripts/verify-packed-manifests.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/verify-packed-manifests.mjs b/scripts/verify-packed-manifests.mjs index c24408182fb..bc57ed57d78 100644 --- a/scripts/verify-packed-manifests.mjs +++ b/scripts/verify-packed-manifests.mjs @@ -479,7 +479,7 @@ function writeConsumerFixture(packDir, packedWorkspaces) { export const CONSUMER_MIN_RELEASE_AGE_SECONDS = 24 * 60 * 60; const BLOCKED_BY_AGE = - /No version matching "([^"]+)" found for specifier "[^"]*" \(blocked by minimum-release-age/g; + /No version matching "[^"]+" found for specifier "([^"]+)" \(blocked by minimum-release-age/g; export function installConsumer(dir, { args = [], env = process.env } = {}) { const install = () => From e22a831c7fb77073718941f2ef3dcb23daa181d4 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 04:05:31 -0700 Subject: [PATCH 08/12] fix(cli): resend the stems request with its key until the service answers The stems service answers once the stems are made, which can outlast the connection, and answers 409 request_in_progress to a resend while the first request runs. A failed job's reason is failure_message. --- packages/cli/src/audio/read/read.test.ts | 14 ++--- packages/cli/src/audio/read/stems.ts | 67 ++++++++++++++++-------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/packages/cli/src/audio/read/read.test.ts b/packages/cli/src/audio/read/read.test.ts index 778cac5cb98..1be7518f24d 100644 --- a/packages/cli/src/audio/read/read.test.ts +++ b/packages/cli/src/audio/read/read.test.ts @@ -47,9 +47,9 @@ const bed = (t: number) => let server: Server; let stalledPath: string | null = null; let onStalled = () => {}; -let polls = 0; let stemCalls = 0; let stemRequest: unknown = null; +const stemKeys: unknown[] = []; const stemFiles: Record = {}; async function body(req: IncomingMessage) { @@ -74,10 +74,9 @@ const routes = new Map { - polls++; + stemKeys.push(req.headers["idempotency-key"]); + if (stemCalls === 1) + return reply(409, { error: { code: "request_in_progress", message: "Retry shortly." } }); reply(200, { data: { id: "job-1", @@ -177,7 +176,8 @@ describe.skipIf(!ffmpeg)("readAudio", () => { quality: "best", }); expect(stemRequest).toEqual({ version: 1, audio: { type: "asset_id", asset_id: "episode" } }); - expect(polls).toBe(1); + expect(stemKeys).toHaveLength(2); + expect(stemKeys[1]).toBe(stemKeys[0]); expect(analysis).toMatchObject({ separation: "stems", stems: ["bass", "drums", "piano", "vocals"], @@ -193,7 +193,7 @@ describe.skipIf(!ffmpeg)("readAudio", () => { expect(readFileSync(textPath, "utf8")).toContain("effect boom (S1): low"); expect(existsSync(join(cacheDir, "stems"))).toBe(true); await readAudio(path, "episode.wav", { cacheDir, quality: "best" }); - expect(stemCalls).toBe(1); + expect(stemCalls).toBe(2); }); it.each(["/put", "/v3/audio/stems"])( diff --git a/packages/cli/src/audio/read/stems.ts b/packages/cli/src/audio/read/stems.ts index 65613469d78..9e5f6e46c0f 100644 --- a/packages/cli/src/audio/read/stems.ts +++ b/packages/cli/src/audio/read/stems.ts @@ -23,22 +23,23 @@ import { * `{ version: 1, audio: { type: "asset_id", asset_id } }`. The recording is uploaded * first as 192 kb/s stereo MP3 through the same origin's asset direct-upload routes. * - * Response: `{ data: StemsJob }`, or a bare `StemsJob`. While `status` is `pending` or `processing`, - * `GET /` returns the job again. A `completed` job lists one URL per stem; - * the names are vocals, drums, bass, guitar, piano and other. + * Response: `{ data: StemsJob }` once the stems are made, which can take minutes. A failure is an HTTP error + * whose body is `{ error: { code, message } }`. The same key and body resent answer the same job, and 409 + * `request_in_progress` while the first request still runs. A `completed` job lists one URL per stem; the + * names are vocals, drums, bass, guitar, piano and other. */ interface StemsJob { id: string; - status: "pending" | "processing" | "completed" | "failed"; + status: "pending" | "completed" | "failed"; stems?: { name: string; url: string }[]; - error?: { message?: string }; + failure_message?: string; } const URL_ENV = "HYPERFRAMES_AUDIO_STEMS_URL"; const KEY_ENV = "HYPERFRAMES_AUDIO_API_KEY"; const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS"; const DEFAULT_TIMEOUT_MS = 30 * 60_000; -const POLL_MS = 5000; +const RETRY_MS = 5000; const DONE = "stems.json"; const STEM_NAMES = ["vocals", "drums", "bass", "guitar", "piano", "other"] as const; @@ -62,7 +63,25 @@ export function cachedStems(dir: string): Record | null { return Object.values(paths).every((p) => existsSync(p)) ? paths : null; } +class StillRunning extends Error {} + +const DROPPED = new Set(["UND_ERR_HEADERS_TIMEOUT", "UND_ERR_SOCKET"]); + +const dropped = (error: unknown) => + error instanceof TypeError && + DROPPED.has((error.cause as { code?: string } | undefined)?.code ?? ""); + +async function stillRunning(response: Response): Promise { + if (response.status !== 409) return false; + const body = (await response + .clone() + .json() + .catch(() => null)) as { error?: { code?: string } } | null; + return body?.error?.code === "request_in_progress"; +} + async function job(response: Response): Promise { + if (await stillRunning(response)) throw new StillRunning(); if (!response.ok) throw await serviceFailure(response, "the stems service"); const body = (await response.json()) as { data?: StemsJob } | StemsJob | null; const value = (body && "data" in body ? body.data : body) as StemsJob | undefined; @@ -82,6 +101,21 @@ async function encoded(path: string, dir: string, signal: AbortSignal): Promise< return out; } +async function separated( + url: string, + request: RequestInit, + signal: AbortSignal, +): Promise { + for (;;) { + try { + return await job(await fetch(url, { ...request, signal })); + } catch (error) { + if (!(error instanceof StillRunning || dropped(error))) throw error; + await sleep(RETRY_MS, undefined, { signal }); + } + } +} + /** Separate the recording through the configured service and save each stem into `dir`. */ export async function fetchStems( path: string, @@ -106,10 +140,10 @@ export async function fetchStems( idempotencyKey: randomUUID(), signal, }); - let current = await job( - await fetch(url, { + const current = await separated( + url, + { method: "POST", - signal, redirect: "error", headers: { ...headers, @@ -117,21 +151,12 @@ export async function fetchStems( "content-type": "application/json", }, body: JSON.stringify({ version: 1, audio: { type: "asset_id", asset_id: asset.asset_id } }), - }), + }, + signal, ); - while (current.status === "pending" || current.status === "processing") { - await sleep(POLL_MS, undefined, { signal }); - current = await job( - await fetch(`${url.replace(/\/$/, "")}/${encodeURIComponent(current.id)}`, { - redirect: "error", - headers, - signal, - }), - ); - } if (current.status !== "completed") throw new Error( - `stem separation failed${current.error?.message ? `: ${current.error.message}` : ""}`, + `stem separation failed${current.failure_message ? `: ${current.failure_message}` : ""}`, ); const stems = current.stems ?? []; if (!stems.some((s) => s.name === VOICE_STEM)) From 99048fecf14a9d27fdaca67de6e4baeb72dcb96d Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 04:06:36 -0700 Subject: [PATCH 09/12] fix(cli): retry incomplete stem separations --- packages/cli/src/audio/read/read.test.ts | 32 ++++++++++++++++++++++-- packages/cli/src/audio/read/read.ts | 1 + packages/cli/src/audio/read/stems.ts | 10 +++++--- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/audio/read/read.test.ts b/packages/cli/src/audio/read/read.test.ts index 778cac5cb98..043de593979 100644 --- a/packages/cli/src/audio/read/read.test.ts +++ b/packages/cli/src/audio/read/read.test.ts @@ -6,7 +6,7 @@ import { join } from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { readAudio } from "./read.js"; -import { fetchStems } from "./stems.js"; +import { cachedStems, fetchStems } from "./stems.js"; const RATE = 22_050; const dir = mkdtempSync(join(tmpdir(), "hf-audio-read-")); @@ -168,6 +168,8 @@ describe.skipIf(!ffmpeg)("readAudio", () => { stemFiles.drums = wav("drums.wav", 8, boom); stemFiles.piano = wav("piano.wav", 8, bed); stemFiles.bass = wav("bass.wav", 8, () => 0); + stemFiles.guitar = stemFiles.bass; + stemFiles.other = stemFiles.bass; stemFiles["../../escape"] = stemFiles.bass; const path = wav("episode.wav", 8, (t) => voiced(t) + boom(t) + bed(t)); vi.stubEnv("HEYGEN_API_KEY", "test-key"); @@ -180,7 +182,7 @@ describe.skipIf(!ffmpeg)("readAudio", () => { expect(polls).toBe(1); expect(analysis).toMatchObject({ separation: "stems", - stems: ["bass", "drums", "piano", "vocals"], + stems: ["bass", "drums", "guitar", "other", "piano", "vocals"], warnings: [expect.stringContaining("no transcript")], }); expect(analysis.voice!.source).toBe("activity"); @@ -196,6 +198,32 @@ describe.skipIf(!ffmpeg)("readAudio", () => { expect(stemCalls).toBe(1); }); + it("falls back on partial stems and retries instead of caching unmeasured layers as silence", async () => { + for (const name of Object.keys(stemFiles)) delete stemFiles[name]; + stemFiles.vocals = wav("partial-vocals.wav", 2, voiced); + const path = wav("partial.wav", 2, voiced); + vi.stubEnv("HEYGEN_API_KEY", "test-key"); + const cacheDir = join(dir, "cache-partial"); + const partial = await readAudio(path, "partial.wav", { cacheDir, quality: "best" }); + expect(partial.analysis).toMatchObject({ separation: "mix", effects: null, music: null }); + expect(partial.analysis.warnings.join("\n")).toContain("incomplete separation: missing drums"); + const callsBefore = stemCalls; + const silence = wav("partial-silence.wav", 2, () => 0); + for (const name of ["drums", "bass", "guitar", "piano", "other"]) stemFiles[name] = silence; + const complete = await readAudio(path, "partial.wav", { cacheDir, quality: "best" }); + expect(complete.cached).toBe(false); + expect(complete.analysis.separation).toBe("stems"); + expect(stemCalls).toBe(callsBefore + 1); + + const stemDir = join(cacheDir, "stems", complete.analysis.sha256.slice(0, 12)); + writeFileSync(join(stemDir, "stems.json"), JSON.stringify({ vocals: "vocals.audio" })); + expect(cachedStems(stemDir)).toBeNull(); + const repaired = await readAudio(path, "partial.wav", { cacheDir, quality: "best" }); + expect(repaired.cached).toBe(false); + expect(stemCalls).toBe(callsBefore + 2); + expect(Object.keys(cachedStems(stemDir)!)).toHaveLength(6); + }); + it.each(["/put", "/v3/audio/stems"])( "aborts a stalled %s request at the separation deadline", async (endpoint) => { diff --git a/packages/cli/src/audio/read/read.ts b/packages/cli/src/audio/read/read.ts index 7321f3ccff8..e8f9c0c03a0 100644 --- a/packages/cli/src/audio/read/read.ts +++ b/packages/cli/src/audio/read/read.ts @@ -194,6 +194,7 @@ export async function readAudio( quality === "best" && (cachedStems(stemDir) !== null || typeof (await stemsAccess()) !== "string"); if ( + (entry.analysis.separation !== "stems" || cachedStems(stemDir) !== null) && entry.analysis.version === READ_VERSION && entry.transcriptKey === transcriptKey && !stemsNowPossible diff --git a/packages/cli/src/audio/read/stems.ts b/packages/cli/src/audio/read/stems.ts index 65613469d78..93dff966958 100644 --- a/packages/cli/src/audio/read/stems.ts +++ b/packages/cli/src/audio/read/stems.ts @@ -58,7 +58,8 @@ export function cachedStems(dir: string): Record | null { const done = join(dir, DONE); if (!existsSync(done)) return null; const names = JSON.parse(readFileSync(done, "utf8")) as Record; - const paths = Object.fromEntries(Object.entries(names).map(([n, f]) => [n, join(dir, f)])); + if (!STEM_NAMES.every((name) => typeof names[name] === "string")) return null; + const paths = Object.fromEntries(STEM_NAMES.map((name) => [name, join(dir, names[name]!)])); return Object.values(paths).every((p) => existsSync(p)) ? paths : null; } @@ -134,8 +135,11 @@ export async function fetchStems( `stem separation failed${current.error?.message ? `: ${current.error.message}` : ""}`, ); const stems = current.stems ?? []; - if (!stems.some((s) => s.name === VOICE_STEM)) - throw new Error("the stems service returned no vocals stem"); + const missing = STEM_NAMES.filter((name) => !stems.some((stem) => stem.name === name)); + if (missing.length) + throw new Error( + `the stems service returned an incomplete separation: missing ${missing.join(", ")}`, + ); mkdirSync(dir, { recursive: true }); const names: Record = {}; for (const name of STEM_NAMES) { From 2f7c4702724cad90c42306aab10330ed546f2472 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 04:20:20 -0700 Subject: [PATCH 10/12] test: validate audio mock route handlers before dispatch --- packages/cli/src/audio/read/read.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/audio/read/read.test.ts b/packages/cli/src/audio/read/read.test.ts index ec0fecd1336..c6145f18039 100644 --- a/packages/cli/src/audio/read/read.test.ts +++ b/packages/cli/src/audio/read/read.test.ts @@ -101,7 +101,7 @@ beforeAll(async () => { const reply: Reply = (status, value) => res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(value)); const route = routes.get(`${req.method} ${req.url}`); - if (route) return route(reply, req, bytes); + if (typeof route === "function") return route(reply, req, bytes); const stem = req.url?.match(/^\/stem\/(\w+)\.wav$/)?.[1]; if (stem && stemFiles[stem]) return res.writeHead(200).end(readFileSync(stemFiles[stem])); res.writeHead(404).end(); From 596a664aea83fa520fdd264aa07b201374dc60f7 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 05:12:29 -0700 Subject: [PATCH 11/12] test: trace premature Windows CLI runner exits --- packages/cli/vitest.config.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index f751f023391..5e6297f06c1 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -8,6 +8,16 @@ import "./scripts/test-home.mjs"; // Windows: sharp's first text render builds Fontconfig's cache for every OS font (about 9 s on a // fresh runner). Set here, before workers fork, because an in-process env write never reaches it. if (process.platform === "win32") { + const exit = process.exit; + process.exit = (code) => { + console.error(new Error(`[DEBUG-6056] runner ${process.pid} explicit exit ${code}`).stack); + return exit(code); + }; + process.on("uncaughtExceptionMonitor", (error) => + console.error("[DEBUG-6056] runner uncaught exception", error), + ); + process.on("exit", (code) => console.error(`[DEBUG-6056] runner ${process.pid} exit ${code}`)); + console.error(`[DEBUG-6056] runner ${process.pid} parent ${process.ppid}`); const dir = mkdtempSync(join(tmpdir(), "hf-vitest-fontconfig-")); process.once("exit", () => rmSync(dir, { recursive: true, force: true })); copyFileSync( From e45dd4e6ad42905335be37de80d9139f9adb45e5 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Thu, 22 Oct 2026 05:29:33 -0700 Subject: [PATCH 12/12] test: remove temporary Windows runner diagnostics --- packages/cli/vitest.config.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 5e6297f06c1..f751f023391 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -8,16 +8,6 @@ import "./scripts/test-home.mjs"; // Windows: sharp's first text render builds Fontconfig's cache for every OS font (about 9 s on a // fresh runner). Set here, before workers fork, because an in-process env write never reaches it. if (process.platform === "win32") { - const exit = process.exit; - process.exit = (code) => { - console.error(new Error(`[DEBUG-6056] runner ${process.pid} explicit exit ${code}`).stack); - return exit(code); - }; - process.on("uncaughtExceptionMonitor", (error) => - console.error("[DEBUG-6056] runner uncaught exception", error), - ); - process.on("exit", (code) => console.error(`[DEBUG-6056] runner ${process.pid} exit ${code}`)); - console.error(`[DEBUG-6056] runner ${process.pid} parent ${process.ppid}`); const dir = mkdtempSync(join(tmpdir(), "hf-vitest-fontconfig-")); process.once("exit", () => rmSync(dir, { recursive: true, force: true })); copyFileSync(