From 12986a11706d9a1333f985b145b9e507a15e94c7 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Tue, 17 Nov 2026 21:46:38 +0200 Subject: [PATCH 1/2] fix(`LlamaEmbeddingContext`): automatically set the batch size for non-causal embedding models --- src/evaluator/LlamaContext/LlamaContext.ts | 7 ++++- src/evaluator/LlamaContext/types.ts | 4 +-- .../utils/resolveContextBatchSizeOption.ts | 13 +++++++++ src/evaluator/LlamaEmbeddingContext.ts | 18 +++++++++++-- src/gguf/insights/GgufInsights.ts | 27 ++++++++++++------- .../GgufInsightsConfigurationResolver.ts | 5 ++++ .../utils/resolveContextContextSizeOption.ts | 10 +++---- .../utils/resolveModelGpuLayersOption.ts | 8 ++++-- src/utils/utilTypes.ts | 8 ++++++ .../nomicEmbedText/embedding.test.ts | 22 +++++++++++++++ 10 files changed, 101 insertions(+), 21 deletions(-) create mode 100644 src/evaluator/LlamaContext/utils/resolveContextBatchSizeOption.ts diff --git a/src/evaluator/LlamaContext/LlamaContext.ts b/src/evaluator/LlamaContext/LlamaContext.ts index 28cd9061..2d483cc1 100644 --- a/src/evaluator/LlamaContext/LlamaContext.ts +++ b/src/evaluator/LlamaContext/LlamaContext.ts @@ -26,6 +26,7 @@ import {resolveBatchItemsPrioritizationStrategy} from "./utils/resolveBatchItems import {LlamaSampler} from "./LlamaSampler.js"; import {TokenPredictor} from "./TokenPredictor.js"; import {padSafeContextSize} from "./utils/padSafeContextSize.js"; +import {resolveContextBatchSizeOption} from "./utils/resolveContextBatchSizeOption.js"; import {LlamaContextSequenceCheckpoints} from "./LlamaContextSequenceCheckpoints.js"; import type {Llama} from "../../bindings/Llama.js"; @@ -973,7 +974,11 @@ export class LlamaContext { contextSize = paddedContextSize; async function createContext(contextSize: number) { - const batchSize = options.batchSize ?? getDefaultContextBatchSize({contextSize, sequences}); + const batchSize = resolveContextBatchSizeOption({ + batchSize: options.batchSize, + contextSize, + sequences + }); const resourceRequirementsEstimation = await _model.fileInsights.estimateContextResourceRequirementsV2({ contextSize, sequences, diff --git a/src/evaluator/LlamaContext/types.ts b/src/evaluator/LlamaContext/types.ts index adcaf169..82ec574b 100644 --- a/src/evaluator/LlamaContext/types.ts +++ b/src/evaluator/LlamaContext/types.ts @@ -1,4 +1,4 @@ -import type {PickOptions} from "../../utils/utilTypes.js"; +import type {InternalType, PickOptions} from "../../utils/utilTypes.js"; import type {GgmlType} from "../../gguf/types/GgufTensorInfoTypes.js"; import type {LlamaGrammarEvaluationState} from "../LlamaGrammarEvaluationState.js"; import type {TokenBias} from "../TokenBias.js"; @@ -46,7 +46,7 @@ export type LlamaContextOptions = { * * Defaults to `512` or `contextSize` if `contextSize` is less than `512`. */ - batchSize?: number, + batchSize?: number | InternalType<"_matchContext">, /** * Flash attention is an optimization in the attention mechanism that makes inference faster, more efficient and uses less memory. diff --git a/src/evaluator/LlamaContext/utils/resolveContextBatchSizeOption.ts b/src/evaluator/LlamaContext/utils/resolveContextBatchSizeOption.ts new file mode 100644 index 00000000..5a3c8c47 --- /dev/null +++ b/src/evaluator/LlamaContext/utils/resolveContextBatchSizeOption.ts @@ -0,0 +1,13 @@ +import {getDefaultContextBatchSize} from "../LlamaContext.js"; +import {padSafeContextSize} from "./padSafeContextSize.js"; +import type {LlamaContextOptions} from "../types.js"; + +export function resolveContextBatchSizeOption({batchSize, contextSize, sequences}: { + batchSize?: LlamaContextOptions["batchSize"], + contextSize: number, + sequences: number +}): number { + return batchSize === "_matchContext" + ? padSafeContextSize(contextSize, "up") + : batchSize ?? getDefaultContextBatchSize({contextSize, sequences}); +} diff --git a/src/evaluator/LlamaEmbeddingContext.ts b/src/evaluator/LlamaEmbeddingContext.ts index b814b034..725a57d5 100644 --- a/src/evaluator/LlamaEmbeddingContext.ts +++ b/src/evaluator/LlamaEmbeddingContext.ts @@ -3,6 +3,7 @@ import {Token} from "../types.js"; import {LlamaText} from "../utils/LlamaText.js"; import {tokenizeInput} from "../utils/tokenizeInput.js"; import {resolveBeginningTokenToPrepend, resolveEndTokenToAppend} from "../utils/tokenizerUtils.js"; +import {LlamaLogLevel} from "../bindings/types.js"; import {LlamaEmbedding} from "./LlamaEmbedding.js"; import type {LlamaModel} from "./LlamaModel/LlamaModel.js"; import type {LlamaContext, LlamaContextSequence} from "./LlamaContext/LlamaContext.js"; @@ -25,7 +26,11 @@ export type LlamaEmbeddingContextOptions = { max?: number }, - /** prompt processing batch size */ + /** + * Prompt processing batch size. + * + * For non-causal models, always matches the resolved context size regardless of the specified batch size. + */ batchSize?: number, /** @@ -163,15 +168,24 @@ export class LlamaEmbeddingContext { if (_model.fileInsights.hasEncoder && _model.fileInsights.hasDecoder) throw new Error("Computing embeddings is not supported for encoder-decoder models."); + const isNonCausal = _model.fileInfo.architectureMetadata?.attention?.causal === false; const llamaContext = await _model.createContext({ contextSize, - batchSize, + batchSize: isNonCausal + ? "_matchContext" + : batchSize, threads, createSignal, ignoreMemorySafetyChecks, _embeddings: true }); + if (isNonCausal && batchSize != null && batchSize !== llamaContext.batchSize) + _model._llama._log(LlamaLogLevel.warn, + `Ignoring batchSize ${batchSize} because this non-causal embedding model requires the batch size to match ` + + `the resolved context size (${llamaContext.batchSize}).` + ); + return new LlamaEmbeddingContext({ _llamaContext: llamaContext }); diff --git a/src/gguf/insights/GgufInsights.ts b/src/gguf/insights/GgufInsights.ts index 534f4101..49dda7a7 100644 --- a/src/gguf/insights/GgufInsights.ts +++ b/src/gguf/insights/GgufInsights.ts @@ -3,12 +3,13 @@ import bytes from "bytes"; import {Llama} from "../../bindings/Llama.js"; import {doesLlamaBackendNeedAddonInitLock, LlamaLocks, LlamaLogLevel} from "../../bindings/types.js"; import {getLlamaWithoutBackend} from "../../bindings/utils/getLlamaWithoutBackend.js"; -import {getDefaultContextBatchSize, getDefaultContextSequences} from "../../evaluator/LlamaContext/LlamaContext.js"; +import {getDefaultContextSequences} from "../../evaluator/LlamaContext/LlamaContext.js"; import {GgufFileInfo} from "../types/GgufFileInfoTypes.js"; import {GgmlType, GgufTensorInfo} from "../types/GgufTensorInfoTypes.js"; import {GgufArchitectureType} from "../types/GgufMetadataTypes.js"; import {getReadablePath} from "../../cli/utils/getReadablePath.js"; import {padSafeContextSize} from "../../evaluator/LlamaContext/utils/padSafeContextSize.js"; +import {resolveContextBatchSizeOption} from "../../evaluator/LlamaContext/utils/resolveContextBatchSizeOption.js"; import {removeNullFields, removeUndefinedFields} from "../../utils/removeNullFields.js"; import {LruCache} from "../../utils/LruCache.js"; import {DisposalPreventionHandle, DisposeGuard} from "../../utils/DisposeGuard.js"; @@ -371,12 +372,16 @@ export class GgufInsights { swaFullCache = false, kvCacheKeyType = GgmlType.F16, kvCacheValueType = GgmlType.F16 }: { - contextSize: number, modelGpuLayers: number, batchSize?: number, sequences?: number, isEmbeddingContext?: boolean, - flashAttention?: boolean, includeGraphOverhead?: boolean, swaFullCache?: boolean, + contextSize: number, modelGpuLayers: number, batchSize?: LlamaContextOptions["batchSize"], sequences?: number, + isEmbeddingContext?: boolean, flashAttention?: boolean, includeGraphOverhead?: boolean, swaFullCache?: boolean, kvCacheKeyType?: GgmlType, kvCacheValueType?: GgmlType }): GgufInsightsResourceRequirements { if (sequences == null) sequences = getDefaultContextSequences(); - if (batchSize == null) batchSize = getDefaultContextBatchSize({contextSize, sequences}); + batchSize = resolveContextBatchSizeOption({ + batchSize, + contextSize, + sequences + }); const llmData = this._ggufFileInfo.architectureMetadata; const tensorInfo = this._ggufFileInfo.fullTensorInfo ?? []; @@ -715,8 +720,8 @@ export class GgufInsights { } public async estimateContextResourceRequirementsV2(options: { - contextSize: number, modelGpuLayers: number, batchSize?: number, sequences?: number, isEmbeddingContext?: boolean, - flashAttention?: LlamaContextOptions["flashAttention"], swaFullCache?: boolean, + contextSize: number, modelGpuLayers: number, batchSize?: LlamaContextOptions["batchSize"], sequences?: number, + isEmbeddingContext?: boolean, flashAttention?: LlamaContextOptions["flashAttention"], swaFullCache?: boolean, kvCacheKeyType?: GgmlType, kvCacheValueType?: GgmlType, useMmap?: boolean, @@ -780,13 +785,17 @@ export class GgufInsights { kvCacheKeyType = GgmlType.F16, kvCacheValueType = GgmlType.F16, simulatorSession = this._simulationSession }: { - contextSize: number, modelGpuLayers: number, batchSize?: number, sequences?: number, isEmbeddingContext?: boolean, - flashAttention?: LlamaContextOptions["flashAttention"], swaFullCache?: boolean, useMmap?: boolean, + contextSize: number, modelGpuLayers: number, batchSize?: LlamaContextOptions["batchSize"], sequences?: number, + isEmbeddingContext?: boolean, flashAttention?: LlamaContextOptions["flashAttention"], swaFullCache?: boolean, useMmap?: boolean, kvCacheKeyType?: GgmlType, kvCacheValueType?: GgmlType, simulatorSession?: GgufInsightsSimulatorSession }): Promise { if (sequences == null) sequences = getDefaultContextSequences(); - if (batchSize == null) batchSize = getDefaultContextBatchSize({contextSize, sequences}); + batchSize = resolveContextBatchSizeOption({ + batchSize, + contextSize, + sequences + }); const kvUnified = false; diff --git a/src/gguf/insights/GgufInsightsConfigurationResolver.ts b/src/gguf/insights/GgufInsightsConfigurationResolver.ts index 9688bb85..5f808079 100644 --- a/src/gguf/insights/GgufInsightsConfigurationResolver.ts +++ b/src/gguf/insights/GgufInsightsConfigurationResolver.ts @@ -208,6 +208,9 @@ export class GgufInsightsConfigurationResolver { : useMmap; let gpuLayersFitMemory = false; await using simulatorSession = this._ggufInsights._createSimulatorSession(); + const batchSize = embeddingContext && this._ggufInsights.ggufFileInfo.architectureMetadata?.attention?.causal === false + ? "_matchContext" + : undefined; try { const layersResolution = await this.resolveModelGpuLayersV2( @@ -266,6 +269,7 @@ export class GgufInsightsConfigurationResolver { try { resolvedContextSize = await this.resolveContextContextSize("auto", { + batchSize, getVramState: async () => ({ total: vramState.total, free: Math.max(0, vramState.free - estimatedModelResourceUsage.gpuVram), @@ -321,6 +325,7 @@ export class GgufInsightsConfigurationResolver { const estimatedContextResourceUsage = await this._ggufInsights.estimateContextResourceRequirementsV2({ contextSize: resolvedContextSize, + batchSize, isEmbeddingContext: embeddingContext, modelGpuLayers: resolvedGpuLayers, flashAttention, diff --git a/src/gguf/insights/utils/resolveContextContextSizeOption.ts b/src/gguf/insights/utils/resolveContextContextSizeOption.ts index f1ccf5e1..5bdfac37 100644 --- a/src/gguf/insights/utils/resolveContextContextSizeOption.ts +++ b/src/gguf/insights/utils/resolveContextContextSizeOption.ts @@ -2,7 +2,7 @@ import {LlamaContextOptions} from "../../../evaluator/LlamaContext/types.js"; import {GgufInsights, GgufInsightsSimulatorSession} from "../GgufInsights.js"; import {BuildGpu} from "../../../bindings/types.js"; import {minAllowedContextSizeInCalculations} from "../../../config.js"; -import {getDefaultContextBatchSize, getDefaultModelContextSize} from "../../../evaluator/LlamaContext/LlamaContext.js"; +import {getDefaultModelContextSize} from "../../../evaluator/LlamaContext/LlamaContext.js"; import {InsufficientMemoryError} from "../../../utils/InsufficientMemoryError.js"; import {getRamUsageFromUnifiedVram} from "./getRamUsageFromUnifiedVram.js"; import type {GgmlType} from "../../types/GgufTensorInfoTypes.js"; @@ -37,7 +37,7 @@ export async function resolveContextContextSizeOption(options: { kvCacheKeyType, kvCacheValueType, swaFullCache, useMmap = modelFileInsights._llama.supportsMmap, getVramState, getRamState, getSwapState, ignoreMemorySafetyChecks = false, isEmbeddingContext = false, maxContextSizeSwapUse = defaultMaxContextSizeSwapUse, - + simulatorSession: _simulatorSession, ramCapIsSet = false, vramCapIsSet = false @@ -67,7 +67,7 @@ export async function resolveContextContextSizeOption(options: { ]); const contextResourceRequirements = await modelFileInsights.estimateContextResourceRequirementsV2({ contextSize: resolvedContextSize, - batchSize: batchSize ?? getDefaultContextBatchSize({contextSize: resolvedContextSize, sequences}), + batchSize, modelGpuLayers: modelGpuLayers, sequences, flashAttention, @@ -118,7 +118,7 @@ export async function resolveContextContextSizeOption(options: { for (let testContextSize = maxContextSize; testContextSize >= minContextSize && testContextSize <= maxContextSize;) { const contextResourceRequirements = await modelFileInsights.estimateContextResourceRequirementsV2({ contextSize: testContextSize, - batchSize: batchSize ?? getDefaultContextBatchSize({contextSize: testContextSize, sequences}), + batchSize, modelGpuLayers: modelGpuLayers, sequences, flashAttention, @@ -175,7 +175,7 @@ export async function resolveContextContextSizeOption(options: { const minContextSizeResourceRequirements = await modelFileInsights.estimateContextResourceRequirementsV2({ contextSize: minContextSize, - batchSize: batchSize ?? getDefaultContextBatchSize({contextSize: minContextSize, sequences}), + batchSize, modelGpuLayers: modelGpuLayers, sequences, flashAttention, diff --git a/src/gguf/insights/utils/resolveModelGpuLayersOption.ts b/src/gguf/insights/utils/resolveModelGpuLayersOption.ts index e1515b23..9c2e64e8 100644 --- a/src/gguf/insights/utils/resolveModelGpuLayersOption.ts +++ b/src/gguf/insights/utils/resolveModelGpuLayersOption.ts @@ -408,7 +408,9 @@ async function getVramRequiredForGpuLayers({ if (fitContext != null && fitContext.contextSize != null) { const contextVram = (await ggufInsights.estimateContextResourceRequirementsV2({ contextSize: fitContext.contextSize, - batchSize: getDefaultContextBatchSize({contextSize: fitContext.contextSize, sequences: 1}), + batchSize: fitContext.embeddingContext && ggufInsights.ggufFileInfo.architectureMetadata?.attention?.causal === false + ? "_matchContext" + : getDefaultContextBatchSize({contextSize: fitContext.contextSize, sequences: 1}), modelGpuLayers: gpuLayers, sequences: 1, isEmbeddingContext: fitContext.embeddingContext ?? false, @@ -476,7 +478,9 @@ async function findMaxPossibleContextSizeForVram({ async test(contextSize) { const contextVram = (await ggufInsights.estimateContextResourceRequirementsV2({ contextSize, - batchSize: getDefaultContextBatchSize({contextSize, sequences: 1}), + batchSize: isEmbeddingContext && ggufInsights.ggufFileInfo.architectureMetadata?.attention?.causal === false + ? "_matchContext" + : getDefaultContextBatchSize({contextSize, sequences: 1}), modelGpuLayers: gpuLayers, sequences: 1, isEmbeddingContext, diff --git a/src/utils/utilTypes.ts b/src/utils/utilTypes.ts index 53211fd5..87e1415b 100644 --- a/src/utils/utilTypes.ts +++ b/src/utils/utilTypes.ts @@ -2,6 +2,14 @@ export type Writable = { -readonly [P in keyof T]: T[P]; }; +export type InternalType = { + /** + * Resolves to `T` in source and `never` in built declaration files + * @internal + */ + type: T +} extends {type: infer Type} ? Type : never; + /** * Omit all the keys from `Value` that are not present in `Options` and are `true`. * diff --git a/test/modelDependent/nomicEmbedText/embedding.test.ts b/test/modelDependent/nomicEmbedText/embedding.test.ts index a0c4848b..4cbf6276 100644 --- a/test/modelDependent/nomicEmbedText/embedding.test.ts +++ b/test/modelDependent/nomicEmbedText/embedding.test.ts @@ -57,6 +57,28 @@ describe("nomic embed text", () => { expect(helloThereEmbedding2.vector).to.eql(helloThereEmbedding.vector); }); + test("matches the batch size to contextSize 1024 and embeds 600 tokens", {timeout: 1000 * 60 * 60 * 2}, async () => { + const modelPath = await getModelFile("nomic-embed-text-v1.5.Q4_K_M.gguf"); + const llama = await getTestLlama(); + + await using model = await llama.loadModel({modelPath}); + expect(model.fileInfo.architectureMetadata.attention?.causal).toBe(false); + + await using context = await model.createEmbeddingContext({contextSize: 1024}); + expect(context._llamaContext.getAllocatedContextSize()).toBe(1024); + expect(context._llamaContext.batchSize).toBe(1024); + + const tokens = model.tokenize("word ".repeat(600)); + const specialTokenCount = context.calculateInputLength(tokens) - tokens.length; + const input = tokens.slice(0, 600 - specialTokenCount); + expect(context.calculateInputLength(input)).toBe(600); + + const embedding = await context.getEmbeddingFor(input); + expect(embedding.vector).toHaveLength(model.embeddingVectorSize); + expect(embedding.vector.every(Number.isFinite)).toBe(true); + expect(embedding.vector.some((value) => value !== 0)).toBe(true); + }); + test("similarity search", {timeout: 1000 * 60 * 60 * 2}, async () => { const modelPath = await getModelFile("nomic-embed-text-v1.5.Q4_K_M.gguf"); const llama = await getTestLlama(); From 485985718736f01c8fb33c3de018243bfb31321d Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Wed, 18 Nov 2026 19:24:06 +0200 Subject: [PATCH 2/2] test: fix tests --- src/evaluator/LlamaContext/types.ts | 3 ++- .../functionary/functionaryModelGpuLayersOptions.test.ts | 4 ++-- test/modelDependent/llama3/functions.test.ts | 7 ++++--- .../stableCode/stableCodeModelGpuLayersOptions.test.ts | 6 +++--- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/evaluator/LlamaContext/types.ts b/src/evaluator/LlamaContext/types.ts index 82ec574b..91b7e4e0 100644 --- a/src/evaluator/LlamaContext/types.ts +++ b/src/evaluator/LlamaContext/types.ts @@ -46,7 +46,8 @@ export type LlamaContextOptions = { * * Defaults to `512` or `contextSize` if `contextSize` is less than `512`. */ - batchSize?: number | InternalType<"_matchContext">, + batchSize?: number | + /** @internal */ InternalType<"_matchContext">, /** * Flash attention is an optimization in the attention mechanism that makes inference faster, more efficient and uses less memory. diff --git a/test/modelDependent/functionary/functionaryModelGpuLayersOptions.test.ts b/test/modelDependent/functionary/functionaryModelGpuLayersOptions.test.ts index 2be019b6..c93ebdb1 100644 --- a/test/modelDependent/functionary/functionaryModelGpuLayersOptions.test.ts +++ b/test/modelDependent/functionary/functionaryModelGpuLayersOptions.test.ts @@ -1593,8 +1593,8 @@ describe("functionary", () => { totalRam: s1GB * 8, freeRam: s1GB * 8 }); - expect(res.gpuLayers).to.toMatchInlineSnapshot("0"); - expect(res.contextSize).to.toMatchInlineSnapshot("8192"); + expect(res.gpuLayers).to.toMatchInlineSnapshot("9"); + expect(res.contextSize).to.toMatchInlineSnapshot("7168"); expect(res.useMmap).to.toMatchInlineSnapshot("true"); expect(res.contextSize).to.be.gte(contextSize); } diff --git a/test/modelDependent/llama3/functions.test.ts b/test/modelDependent/llama3/functions.test.ts index 8e91ccce..747e1b89 100644 --- a/test/modelDependent/llama3/functions.test.ts +++ b/test/modelDependent/llama3/functions.test.ts @@ -257,9 +257,10 @@ describe("llama 3", () => { const res = await chatSession.prompt("Is an apple more expensive than a banana?", promptOptions); - expect(res).toMatchInlineSnapshot( - '"Let me check the prices for you. According to the prices I checked, an apple is indeed more expensive than a banana. The apple costs $6, while the banana costs $4."' - ); + expect(res).toBeOneOf([ + "Let me check the prices for you. According to the prices I checked, an apple is indeed more expensive than a banana. The apple costs $6, while the banana costs $4.", + "Let me check the prices for you. Based on the prices I just checked, it seems that an apple is indeed more expensive than a banana, with an apple costing $6 and a banana costing $4." + ]); }); }); diff --git a/test/modelDependent/stableCode/stableCodeModelGpuLayersOptions.test.ts b/test/modelDependent/stableCode/stableCodeModelGpuLayersOptions.test.ts index ddc20ba2..57ea6992 100644 --- a/test/modelDependent/stableCode/stableCodeModelGpuLayersOptions.test.ts +++ b/test/modelDependent/stableCode/stableCodeModelGpuLayersOptions.test.ts @@ -619,9 +619,9 @@ describe("stableCode", () => { totalVram: s1GB * 2, freeVram: s1GB * 1 }); - expect(res.gpuLayers).to.toMatchInlineSnapshot("8"); - expect(res.contextSize).to.toMatchInlineSnapshot("6144"); - expect(res.useMmap).to.toMatchInlineSnapshot("false"); + expect(res.gpuLayers).to.toMatchInlineSnapshot("0"); + expect(res.contextSize).to.toMatchInlineSnapshot("16384"); + expect(res.useMmap).to.toMatchInlineSnapshot("true"); expect(res.contextSize).to.be.gte(contextSize); } {