From cf635ebf690e01c07f613f8bfd02767c967e92ab Mon Sep 17 00:00:00 2001 From: "Tomwang.eth" Date: Wed, 18 Nov 2026 19:13:31 +0000 Subject: [PATCH] fix: prevent token prediction from corrupting generation on hybrid and recurrent models Hybrid and recurrent models keep the recurrent state only for the latest position, so `llama_memory_seq_pos_min` always points at the latest token. The KV cache position check in `_eraseContextTokenRanges` treated every tail erase as unrecoverable, and the reevaluation fallback corrupted the sequence state when used during speculative token prediction, which changed the generated output and slowed generation down significantly. Set `n_rs_seq` when creating a context so the recurrent state can be rolled back natively, and let the native removal call decide whether an erase is possible on hybrid and recurrent models (it fails atomically, so the reevaluation fallback still covers the cases it rejects). Closes: #652 --- llama/addon/AddonContext.cpp | 5 ++ src/evaluator/LlamaContext/LlamaContext.ts | 13 ++++- .../qwen3.5-0.8b/tokenPredictor.test.ts | 56 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 test/modelDependent/qwen3.5-0.8b/tokenPredictor.test.ts diff --git a/llama/addon/AddonContext.cpp b/llama/addon/AddonContext.cpp index b4cb759e..8ebb1e02 100644 --- a/llama/addon/AddonContext.cpp +++ b/llama/addon/AddonContext.cpp @@ -581,6 +581,11 @@ AddonContext::AddonContext(const Napi::CallbackInfo& info) : Napi::ObjectWrap 1 && info[1].IsObject()) { const auto options = info[1].As(); diff --git a/src/evaluator/LlamaContext/LlamaContext.ts b/src/evaluator/LlamaContext/LlamaContext.ts index 28cd9061..e1cf95ec 100644 --- a/src/evaluator/LlamaContext/LlamaContext.ts +++ b/src/evaluator/LlamaContext/LlamaContext.ts @@ -1444,9 +1444,18 @@ export class LlamaContextSequence { const minKvCachePosition = (this._contextTokens.length === 0 && this._loadedTokenPredictions.length === 0) ? 0 : Math.max(0, this._context._ctx.getSequenceKvCacheMinPosition(this._sequenceId)); - if (resolvedRanges[0] != null && resolvedRanges[0].start <= minKvCachePosition) - // we have to drop the cache and reevaluate the sequence due to missing KV cache + if (resolvedRanges[0] != null && resolvedRanges[0].start <= minKvCachePosition && + !(this.model.fileInsights.isHybrid || this.model.fileInsights.isRecurrent) + ) { + // we have to drop the cache and reevaluate the sequence due to missing KV cache. + // hybrid and recurrent models are excluded from this check, since their minimal KV cache position + // always points at the latest token (the recurrent state is only kept for the current position), + // so every tail erase would be considered unrecoverable here. + // instead, we let the native `seq_rm` call decide whether the removal is possible - + // it can roll back the recurrent state using the `n_rs_seq` snapshots, + // and it fails atomically when it can't, so the fallback below still handles that case. deletionSuccessful = false; + } const tokenPredictionsToRemove = (resolvedRanges.length > 0 && canRemovePredictionTokens) ? this._loadedTokenPredictions.length diff --git a/test/modelDependent/qwen3.5-0.8b/tokenPredictor.test.ts b/test/modelDependent/qwen3.5-0.8b/tokenPredictor.test.ts new file mode 100644 index 00000000..fc6ef39c --- /dev/null +++ b/test/modelDependent/qwen3.5-0.8b/tokenPredictor.test.ts @@ -0,0 +1,56 @@ +import {describe, expect, test} from "vitest"; +import {InputLookupTokenPredictor, LlamaCompletion} from "../../../src/index.js"; +import {getModelFile} from "../../utils/modelFiles.js"; +import {getTestLlama} from "../../utils/getTestLlama.js"; + +describe("qwen3.5 0.8b", () => { + describe("token predictor", () => { + test("token prediction does not change the generated output", {timeout: 1000 * 60 * 60 * 2}, async () => { + const modelPath = await getModelFile("Qwen3.5-0.8B-Q8_0.gguf"); + const llama = await getTestLlama(); + + const model = await llama.loadModel({ + modelPath + }); + + async function generateText(withTokenPredictor: boolean) { + const context = await model.createContext({ + contextSize: 2048 + }); + + try { + const sequence = context.getSequence( + withTokenPredictor + ? {tokenPredictor: new InputLookupTokenPredictor()} + : {} + ); + const completion = new LlamaCompletion({ + contextSequence: sequence + }); + + const text = await completion.generateCompletion("1, 2, 3,", { + temperature: 0, + maxTokens: 40 + }); + + return { + text, + tokenPredictions: sequence.tokenPredictions + }; + } finally { + await context.dispose(); + } + } + + const expected = await generateText(false); + const withTokenPredictor = await generateText(true); + + // ensure that token prediction was actually used, and that predictions were both validated and refuted, + // so the rollback of refuted predictions was exercised + expect(withTokenPredictor.tokenPredictions.validated).toBeGreaterThan(0); + expect(withTokenPredictor.tokenPredictions.refuted).toBeGreaterThan(0); + + expect(withTokenPredictor.text).to.eql(expected.text); + }); + }); +});