Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions llama/addon/AddonContext.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,11 @@ AddonContext::AddonContext(const Napi::CallbackInfo& info) : Napi::ObjectWrap<Ad
context_params.no_perf = true;
context_params.swa_full = false;
context_params.kv_unified = false;
// enable recurrent state rollback snapshots (used by hybrid/recurrent models) so that
// removing recently-evaluated tokens (e.g. refuted token predictions) can roll back
// the recurrent state instead of failing the removal.
// llama.cpp clamps this to 0 for models that don't support recurrent partial rollback.
context_params.n_rs_seq = 16;

if (info.Length() > 1 && info[1].IsObject()) {
const auto options = info[1].As<Napi::Object>();
Expand Down
13 changes: 11 additions & 2 deletions src/evaluator/LlamaContext/LlamaContext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1444,9 +1444,18 @@ export class LlamaContextSequence {
const minKvCachePosition = (this._contextTokens.length === 0 && this._loadedTokenPredictions.length === 0)
? 0
: Math.max(0, this._context._ctx.getSequenceKvCacheMinPosition(this._sequenceId));
if (resolvedRanges[0] != null && resolvedRanges[0].start <= minKvCachePosition)
// we have to drop the cache and reevaluate the sequence due to missing KV cache
if (resolvedRanges[0] != null && resolvedRanges[0].start <= minKvCachePosition &&
!(this.model.fileInsights.isHybrid || this.model.fileInsights.isRecurrent)
) {
// we have to drop the cache and reevaluate the sequence due to missing KV cache.
// hybrid and recurrent models are excluded from this check, since their minimal KV cache position
// always points at the latest token (the recurrent state is only kept for the current position),
// so every tail erase would be considered unrecoverable here.
// instead, we let the native `seq_rm` call decide whether the removal is possible -
// it can roll back the recurrent state using the `n_rs_seq` snapshots,
// and it fails atomically when it can't, so the fallback below still handles that case.
deletionSuccessful = false;
}

const tokenPredictionsToRemove = (resolvedRanges.length > 0 && canRemovePredictionTokens)
? this._loadedTokenPredictions.length
Expand Down
56 changes: 56 additions & 0 deletions test/modelDependent/qwen3.5-0.8b/tokenPredictor.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import {describe, expect, test} from "vitest";
import {InputLookupTokenPredictor, LlamaCompletion} from "../../../src/index.js";
import {getModelFile} from "../../utils/modelFiles.js";
import {getTestLlama} from "../../utils/getTestLlama.js";

describe("qwen3.5 0.8b", () => {
describe("token predictor", () => {
test("token prediction does not change the generated output", {timeout: 1000 * 60 * 60 * 2}, async () => {
const modelPath = await getModelFile("Qwen3.5-0.8B-Q8_0.gguf");
const llama = await getTestLlama();

const model = await llama.loadModel({
modelPath
});

async function generateText(withTokenPredictor: boolean) {
const context = await model.createContext({
contextSize: 2048
});

try {
const sequence = context.getSequence(
withTokenPredictor
? {tokenPredictor: new InputLookupTokenPredictor()}
: {}
);
const completion = new LlamaCompletion({
contextSequence: sequence
});

const text = await completion.generateCompletion("1, 2, 3,", {
temperature: 0,
maxTokens: 40
});

return {
text,
tokenPredictions: sequence.tokenPredictions
};
} finally {
await context.dispose();
}
}

const expected = await generateText(false);
const withTokenPredictor = await generateText(true);

// ensure that token prediction was actually used, and that predictions were both validated and refuted,
// so the rollback of refuted predictions was exercised
expect(withTokenPredictor.tokenPredictions.validated).toBeGreaterThan(0);
expect(withTokenPredictor.tokenPredictions.refuted).toBeGreaterThan(0);

expect(withTokenPredictor.text).to.eql(expected.text);
});
});
});