diff --git a/llama/addon/AddonContext.cpp b/llama/addon/AddonContext.cpp index b4cb759e..8ebb1e02 100644 --- a/llama/addon/AddonContext.cpp +++ b/llama/addon/AddonContext.cpp @@ -581,6 +581,11 @@ AddonContext::AddonContext(const Napi::CallbackInfo& info) : Napi::ObjectWrap 1 && info[1].IsObject()) { const auto options = info[1].As(); diff --git a/src/evaluator/LlamaContext/LlamaContext.ts b/src/evaluator/LlamaContext/LlamaContext.ts index 28cd9061..e1cf95ec 100644 --- a/src/evaluator/LlamaContext/LlamaContext.ts +++ b/src/evaluator/LlamaContext/LlamaContext.ts @@ -1444,9 +1444,18 @@ export class LlamaContextSequence { const minKvCachePosition = (this._contextTokens.length === 0 && this._loadedTokenPredictions.length === 0) ? 0 : Math.max(0, this._context._ctx.getSequenceKvCacheMinPosition(this._sequenceId)); - if (resolvedRanges[0] != null && resolvedRanges[0].start <= minKvCachePosition) - // we have to drop the cache and reevaluate the sequence due to missing KV cache + if (resolvedRanges[0] != null && resolvedRanges[0].start <= minKvCachePosition && + !(this.model.fileInsights.isHybrid || this.model.fileInsights.isRecurrent) + ) { + // we have to drop the cache and reevaluate the sequence due to missing KV cache. + // hybrid and recurrent models are excluded from this check, since their minimal KV cache position + // always points at the latest token (the recurrent state is only kept for the current position), + // so every tail erase would be considered unrecoverable here. + // instead, we let the native `seq_rm` call decide whether the removal is possible - + // it can roll back the recurrent state using the `n_rs_seq` snapshots, + // and it fails atomically when it can't, so the fallback below still handles that case. deletionSuccessful = false; + } const tokenPredictionsToRemove = (resolvedRanges.length > 0 && canRemovePredictionTokens) ? this._loadedTokenPredictions.length diff --git a/test/modelDependent/qwen3.5-0.8b/tokenPredictor.test.ts b/test/modelDependent/qwen3.5-0.8b/tokenPredictor.test.ts new file mode 100644 index 00000000..fc6ef39c --- /dev/null +++ b/test/modelDependent/qwen3.5-0.8b/tokenPredictor.test.ts @@ -0,0 +1,56 @@ +import {describe, expect, test} from "vitest"; +import {InputLookupTokenPredictor, LlamaCompletion} from "../../../src/index.js"; +import {getModelFile} from "../../utils/modelFiles.js"; +import {getTestLlama} from "../../utils/getTestLlama.js"; + +describe("qwen3.5 0.8b", () => { + describe("token predictor", () => { + test("token prediction does not change the generated output", {timeout: 1000 * 60 * 60 * 2}, async () => { + const modelPath = await getModelFile("Qwen3.5-0.8B-Q8_0.gguf"); + const llama = await getTestLlama(); + + const model = await llama.loadModel({ + modelPath + }); + + async function generateText(withTokenPredictor: boolean) { + const context = await model.createContext({ + contextSize: 2048 + }); + + try { + const sequence = context.getSequence( + withTokenPredictor + ? {tokenPredictor: new InputLookupTokenPredictor()} + : {} + ); + const completion = new LlamaCompletion({ + contextSequence: sequence + }); + + const text = await completion.generateCompletion("1, 2, 3,", { + temperature: 0, + maxTokens: 40 + }); + + return { + text, + tokenPredictions: sequence.tokenPredictions + }; + } finally { + await context.dispose(); + } + } + + const expected = await generateText(false); + const withTokenPredictor = await generateText(true); + + // ensure that token prediction was actually used, and that predictions were both validated and refuted, + // so the rollback of refuted predictions was exercised + expect(withTokenPredictor.tokenPredictions.validated).toBeGreaterThan(0); + expect(withTokenPredictor.tokenPredictions.refuted).toBeGreaterThan(0); + + expect(withTokenPredictor.text).to.eql(expected.text); + }); + }); +});