From 276a227478fed3f6ce7009227a5bb652665c6c99 Mon Sep 17 00:00:00 2001 From: Barbara Wojtarowicz Date: Fri, 24 Jul 2026 17:58:56 +0200 Subject: [PATCH 1/5] refactor: moved creating seekDecoderDaemon thread to AudioFileSourceNode constructor --- .../core/sources/AudioFileSourceNode.cpp | 73 +++++++++++++++++-- .../core/sources/AudioFileSourceNode.h | 8 ++ .../cpp/audioapi/dsp/WsolaTimeStretcher.cpp | 57 ++++++++++++++- .../cpp/audioapi/dsp/WsolaTimeStretcher.h | 10 +++ .../test/src/utils/WsolaTimeStretcherTest.cpp | 36 +++++++++ 5 files changed, 177 insertions(+), 7 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/sources/AudioFileSourceNode.cpp b/packages/react-native-audio-api/common/cpp/audioapi/core/sources/AudioFileSourceNode.cpp index b78ce4613..c75710fe2 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/sources/AudioFileSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/sources/AudioFileSourceNode.cpp @@ -14,10 +14,12 @@ #include #include #include +#include #include #include #include #include +#include #include #if !RN_AUDIO_API_FFMPEG_DISABLED @@ -134,9 +136,75 @@ bool AudioFileSourceNode::initDecoder( channelCount_, context->getSampleRate()); + startDecoderThread(); + // Wall-clock budget to pull ~50ms of PCM from the decoder (not audio delay). + // Local files typically finish well under this; audio-thread warmup covers misses. + static constexpr size_t kPrimeTimeoutMs = 0; + primeWsolaInputFromDecoder(kPrimeTimeoutMs); return true; } +void AudioFileSourceNode::startDecoderThread() { + if (seekDecoderDaemon_ == nullptr || seekDecoderThread_.joinable()) { + return; + } + + seekDecoderThread_ = std::thread(std::move(*seekDecoderDaemon_)); + seekDecoderDaemon_.reset(); +} + +void AudioFileSourceNode::primeWsolaInputFromDecoder(size_t timeoutMs) { + if (frameReceiver_ == nullptr || playbackRateBuffer_ == nullptr) { + return; + } + + const size_t framesNeeded = + std::max(wsolaStretcher_.getRequiredInputFrames(), wsolaStretcher_.getMinInputFramesToRun()); + if (framesNeeded == 0) { + return; + } + + const size_t chunkCapacity = std::max(framesNeeded, static_cast(RENDER_QUANTUM_SIZE)); + if (!ensurePlaybackRateBufferSize(chunkCapacity)) { + return; + } + + using clock = std::chrono::steady_clock; + const auto deadline = clock::now() + std::chrono::milliseconds(timeoutMs); + + DecoderData chunk; + while (wsolaStretcher_.getBufferedInputFrames() < framesNeeded && clock::now() < deadline) { + if (frameReceiver_->try_receive(chunk) != channels::spsc::ResponseStatus::SUCCESS) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + + if (chunk.state == StreamState::END_OF_STREAM || chunk.state == StreamState::DISCONTINUOUS) { + break; + } + if (chunk.size == 0) { + continue; + } + + if (!ensurePlaybackRateBufferSize(chunk.size)) { + break; + } + + playbackRateBuffer_->zero(); + size_t totalInputFrames = 0; + appendFromInterleaved( + chunk.interleavedBuffer.data(), chunk.size, 0, chunk.size, totalInputFrames); + if (totalInputFrames == 0) { + continue; + } + + if (volume_ != 1.0f) { + playbackRateBuffer_->scale(volume_); + } + wsolaStretcher_.feedInput(*playbackRateBuffer_, totalInputFrames); + } +} + void AudioFileSourceNode::setPlaybackRate(float v) { if (decoderState_ == nullptr) { return; @@ -385,11 +453,6 @@ void AudioFileSourceNode::start(double when) { endOfStreamStopPending_ = false; endOfStreamDrainPending_ = false; positionChanged_.requestFlush(); - - if (seekDecoderDaemon_) { - seekDecoderThread_ = std::thread(std::move(*seekDecoderDaemon_)); - seekDecoderDaemon_.reset(); - } } void AudioFileSourceNode::bindMediaElementSource(uint64_t bindingId) { diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/sources/AudioFileSourceNode.h b/packages/react-native-audio-api/common/cpp/audioapi/core/sources/AudioFileSourceNode.h index c38434494..ee6948d24 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/sources/AudioFileSourceNode.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/sources/AudioFileSourceNode.h @@ -188,6 +188,14 @@ class AudioFileSourceNode : public AudioScheduledSourceNode { const std::shared_ptr &context, AudioFileSourceOptions &options); + /// @brief Starts the SeekDecoderDaemon worker thread if it is not already running. + /// @note JS / construction thread only. + void startDecoderThread(); + + /// @brief Pulls decoded packets into @ref wsolaStretcher_ until primed or @p timeoutMs elapses. + /// @note JS / construction thread only — must run before the audio thread consumes @ref frameReceiver_. + void primeWsolaInputFromDecoder(size_t timeoutMs); + /// @brief Attempts to read the next chunk of decoded frames from the daemon. /// @param outData decoded frames and metadata; only valid if return value is true. /// @return false if no decoded frames are available; true if @p outData is filled and ready to process. diff --git a/packages/react-native-audio-api/common/cpp/audioapi/dsp/WsolaTimeStretcher.cpp b/packages/react-native-audio-api/common/cpp/audioapi/dsp/WsolaTimeStretcher.cpp index 0b005bbe4..06c11b0df 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/dsp/WsolaTimeStretcher.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/dsp/WsolaTimeStretcher.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -71,6 +72,9 @@ void WsolaTimeStretcher::reset() { searchBlockIndex_ = 0; outputReadIndex_ = 0; + firstSampleFound_ = false; + totalFramesOutput_ = 0; + for (auto &channel : inputQueue_) { channel.clear(); } @@ -82,6 +86,25 @@ void WsolaTimeStretcher::reset() { } } +size_t WsolaTimeStretcher::getMinInputFramesToRun() const { + if (windowSize_ == 0 || searchIntervalFrames_ == 0) { + return 0; + } + + // Last search candidate starts at searchBlockIndex_ + searchIntervalFrames_ - 1. + const int lastCandidateStart = searchBlockIndex_ + static_cast(searchIntervalFrames_) - 1; + const int targetNeed = maxSourceIndexForBlock(targetBlockIndex_); + const int searchNeed = maxSourceIndexForBlock(lastCandidateStart); + return static_cast(std::max(targetNeed, searchNeed) + 1); +} + +void WsolaTimeStretcher::feedInput(const DSPAudioBuffer &input, size_t inputFrames) { + if (channels_ == 0 || windowSize_ == 0 || inputFrames == 0) { + return; + } + appendInput(input, inputFrames); +} + void WsolaTimeStretcher::process( const DSPAudioBuffer &input, size_t inputFrames, @@ -114,6 +137,31 @@ void WsolaTimeStretcher::process( availableOutputFrames() < outputFrames * 2) { runOneIteration(playbackRate); } + + if (!firstSampleFound_) { + for (int i = 0; i < output.getNumberOfChannels(); ++i) { + auto *channel = output.getChannel(i); + for (int j = 0; j < channel->getSize(); ++j) { + if (std::abs(channel->operator[](j)) > 1e-6f) { + firstSampleFound_ = true; + + size_t absoluteFrameIndex = totalFramesOutput_ + j; + double latencyMs = (static_cast(absoluteFrameIndex) / sampleRate_) * 1000.0; + + std::cout << "[WSOLA] Pierwszy dzwiek! " + << "Ramka wyjsciowa: " << absoluteFrameIndex << " | Opoznienie: " << latencyMs + << " ms" + << " | Playback Rate: " << playbackRate << std::endl; + + break; + } + } + if (firstSampleFound_) + break; + } + } + + totalFramesOutput_ += outputFrames; } size_t @@ -255,13 +303,18 @@ bool WsolaTimeStretcher::canRunIteration() const { } const int inputFrames = static_cast(inputQueue_[0].size()); - const int searchBlockSize = static_cast(searchIntervalFrames_ + windowSize_ - 1); if (maxSourceIndexForBlock(targetBlockIndex_) >= inputFrames) { return false; } - return maxSourceIndexForBlock(searchBlockIndex_ + searchBlockSize - 1) < inputFrames; + // Last search candidate window starts at searchBlockIndex_ + searchIntervalFrames_ - 1 + // (not searchBlockSize - 1, which double-counted the trailing window). + if (searchIntervalFrames_ == 0) { + return false; + } + const int lastCandidateStart = searchBlockIndex_ + static_cast(searchIntervalFrames_) - 1; + return maxSourceIndexForBlock(lastCandidateStart) < inputFrames; } bool WsolaTimeStretcher::targetIsWithinSearchRegion() const { diff --git a/packages/react-native-audio-api/common/cpp/audioapi/dsp/WsolaTimeStretcher.h b/packages/react-native-audio-api/common/cpp/audioapi/dsp/WsolaTimeStretcher.h index 9af54135b..e158d5008 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/dsp/WsolaTimeStretcher.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/dsp/WsolaTimeStretcher.h @@ -18,6 +18,10 @@ class WsolaTimeStretcher { return searchIntervalFrames_ + windowSize_; } + /// Minimum input frames needed for @ref canRunIteration at the current analysis + /// pointers (search span + target window, accounting for pitchFactor_). + [[nodiscard]] size_t getMinInputFramesToRun() const; + [[nodiscard]] size_t getBufferedInputFrames() const { return inputQueue_.empty() ? 0 : inputQueue_[0].size(); } @@ -26,6 +30,9 @@ class WsolaTimeStretcher { return availableOutputFrames(); } + /// Appends PCM to the analysis queue without rendering output (startup prefill). + void feedInput(const DSPAudioBuffer &input, size_t inputFrames); + void process( const DSPAudioBuffer &input, size_t inputFrames, @@ -70,6 +77,9 @@ class WsolaTimeStretcher { int searchBlockIndex_{0}; size_t outputReadIndex_{0}; + bool firstSampleFound_ = false; + size_t totalFramesOutput_ = 0; + std::vector olaWindow_; std::vector transitionWindow_; std::vector> inputQueue_; diff --git a/packages/react-native-audio-api/common/cpp/test/src/utils/WsolaTimeStretcherTest.cpp b/packages/react-native-audio-api/common/cpp/test/src/utils/WsolaTimeStretcherTest.cpp index 48e84e7ce..f07c5d563 100644 --- a/packages/react-native-audio-api/common/cpp/test/src/utils/WsolaTimeStretcherTest.cpp +++ b/packages/react-native-audio-api/common/cpp/test/src/utils/WsolaTimeStretcherTest.cpp @@ -79,6 +79,42 @@ TEST_F(WsolaTimeStretcherTest, RequiredInputFramesAt44100Hz) { EXPECT_EQ(stretcher_.getRequiredInputFrames(), 2205u); } +TEST_F(WsolaTimeStretcherTest, MinInputFramesToRunMatchesSearchSpanNeed) { + // Cold start: last candidate needs searchInterval + window - 1 source samples. + EXPECT_EQ(stretcher_.getMinInputFramesToRun(), stretcher_.getRequiredInputFrames() - 1u); + EXPECT_LE(stretcher_.getMinInputFramesToRun(), stretcher_.getRequiredInputFrames()); +} + +TEST_F(WsolaTimeStretcherTest, FeedInputPrimesForImmediateFirstQuantumOutput) { + const size_t framesNeeded = stretcher_.getMinInputFramesToRun(); + ASSERT_GT(framesNeeded, 0u); + + DSPAudioBuffer prime(framesNeeded, 1, kSampleRate); + for (size_t i = 0; i < framesNeeded; ++i) { + prime.getChannel(0)->span()[i] = + std::sin(2.0f * PI * 440.0f * static_cast(i) / kSampleRate); + } + stretcher_.feedInput(prime, framesNeeded); + EXPECT_GE(stretcher_.getBufferedInputFrames(), framesNeeded); + + // One quantum of additional input at rate 1.5 — should emit non-zero in the first process(). + const size_t inputFrames = inputFramesForQuantum(kPlaybackRate); + DSPAudioBuffer input(inputFrames, 1, kSampleRate); + DSPAudioBuffer output(kQuantum, 1, kSampleRate); + for (size_t i = 0; i < inputFrames; ++i) { + input.getChannel(0)->span()[i] = + std::sin(2.0f * PI * 440.0f * static_cast(framesNeeded + i) / kSampleRate); + } + + stretcher_.process(input, inputFrames, output, kQuantum, kPlaybackRate); + + float peak = 0.0f; + for (size_t i = 0; i < output.getSize(); ++i) { + peak = std::max(peak, std::abs(output.getChannel(0)->span()[i])); + } + EXPECT_GT(peak, 0.01f); +} + TEST_F(WsolaTimeStretcherTest, SchedulingLatencyFormulaMatchesDocumentedConstants) { EXPECT_NEAR(schedulingLatencySeconds(1.0f), 0.03f, 1e-6f); EXPECT_NEAR(schedulingLatencySeconds(kPlaybackRate), 0.04f, 1e-6f); From 88c4610dc1d2ae7e97ff11a46b792582c74f1b26 Mon Sep 17 00:00:00 2001 From: Barbara Wojtarowicz Date: Wed, 29 Jul 2026 17:44:26 +0200 Subject: [PATCH 2/5] refactor: priming wsola in start, experiments --- .claude/skills/utilities/SKILL.md | 16 + .../src/examples/AudioTag/AudioTag.tsx | 49 +- .../WsolaScheduleSplit/WsolaScheduleSplit.tsx | 694 ++++++++++++++++++ .../src/examples/WsolaScheduleSplit/index.ts | 1 + apps/common-app/src/examples/index.ts | 8 + apps/fabric-example/ios/Podfile.lock | 6 +- .../AudioBufferBaseSourceNodeHostObject.cpp | 2 +- .../HostObjects/utils/NodeOptionsParser.h | 1 + .../sources/AudioBufferBaseSourceNode.cpp | 60 +- .../core/sources/AudioBufferBaseSourceNode.h | 8 + .../core/sources/AudioBufferSourceNode.cpp | 3 + .../core/sources/AudioFileSourceNode.cpp | 23 +- .../core/sources/AudioFileSourceNode.h | 5 +- .../cpp/audioapi/dsp/WsolaTimeStretcher.cpp | 50 +- .../cpp/audioapi/dsp/WsolaTimeStretcher.h | 5 + .../src/core/BaseAudioContext.ts | 7 +- 16 files changed, 868 insertions(+), 70 deletions(-) create mode 100644 apps/common-app/src/examples/WsolaScheduleSplit/WsolaScheduleSplit.tsx create mode 100644 apps/common-app/src/examples/WsolaScheduleSplit/index.ts diff --git a/.claude/skills/utilities/SKILL.md b/.claude/skills/utilities/SKILL.md index 6be5092a4..62ea30cae 100644 --- a/.claude/skills/utilities/SKILL.md +++ b/.claude/skills/utilities/SKILL.md @@ -231,6 +231,22 @@ Higher-level DSP blocks. Read each header before use. --- +### `WsolaTimeStretcher.h` — pitch-preserving time stretch + +Needs ~`getRequiredInputFrames()` of PCM in its internal queue before the first OLA +iteration. + +- **File / AudioTag:** early `SeekDecoderDaemon` fills the SPSC; construction + `primeWsolaInputFromDecoder()` waits until the analysis queue is full (no wall-clock + deadline). No first-quantum warmup — process only feeds the current quantum. +- **Buffer sources:** `AudioBufferSourceNode::start` calls `primeWsolaInput()` after + setting `vReadIndex_` (advances cursor by frames fed). Prefill happens only there; + `processWithPitchCorrection` feeds one quantum's input. Compare cold-start with DC + ones sample + abs/relative first-sound logs in `WsolaTimeStretcher`. Scratch sized + with `scratchBufferFrames(sampleRate)`. + +--- + ### `SpectrumAnalyser.h` — shared windowed-FFT magnitude spectrum Owns FFT scratch state (Blackman window, temp array, complex scratch, magnitude diff --git a/apps/common-app/src/examples/AudioTag/AudioTag.tsx b/apps/common-app/src/examples/AudioTag/AudioTag.tsx index ad4d4ab6b..449d33c8c 100644 --- a/apps/common-app/src/examples/AudioTag/AudioTag.tsx +++ b/apps/common-app/src/examples/AudioTag/AudioTag.tsx @@ -1,11 +1,10 @@ import React, { useCallback, useEffect, - useMemo, useRef, useState, } from 'react'; -import { Text, useWindowDimensions, View } from 'react-native'; +import { useWindowDimensions, View } from 'react-native'; import { Audio, AudioTagHandle, @@ -97,41 +96,25 @@ const AudioTag: React.FC = () => { // console.log('onVolumeChange', volume); }, []); - const audioTagElement = useMemo( - () => ( -