Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/evaluator/LlamaContext/LlamaContext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {resolveBatchItemsPrioritizationStrategy} from "./utils/resolveBatchItems
import {LlamaSampler} from "./LlamaSampler.js";
import {TokenPredictor} from "./TokenPredictor.js";
import {padSafeContextSize} from "./utils/padSafeContextSize.js";
import {resolveContextBatchSizeOption} from "./utils/resolveContextBatchSizeOption.js";
import {LlamaContextSequenceCheckpoints} from "./LlamaContextSequenceCheckpoints.js";
import type {Llama} from "../../bindings/Llama.js";

Expand Down Expand Up @@ -973,7 +974,11 @@ export class LlamaContext {
contextSize = paddedContextSize;

async function createContext(contextSize: number) {
const batchSize = options.batchSize ?? getDefaultContextBatchSize({contextSize, sequences});
const batchSize = resolveContextBatchSizeOption({
batchSize: options.batchSize,
contextSize,
sequences
});
const resourceRequirementsEstimation = await _model.fileInsights.estimateContextResourceRequirementsV2({
contextSize,
sequences,
Expand Down
5 changes: 3 additions & 2 deletions src/evaluator/LlamaContext/types.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type {PickOptions} from "../../utils/utilTypes.js";
import type {InternalType, PickOptions} from "../../utils/utilTypes.js";
import type {GgmlType} from "../../gguf/types/GgufTensorInfoTypes.js";
import type {LlamaGrammarEvaluationState} from "../LlamaGrammarEvaluationState.js";
import type {TokenBias} from "../TokenBias.js";
Expand Down Expand Up @@ -46,7 +46,8 @@ export type LlamaContextOptions = {
*
* Defaults to `512` or `contextSize` if `contextSize` is less than `512`.
*/
batchSize?: number,
batchSize?: number |
/** @internal */ InternalType<"_matchContext">,

/**
* Flash attention is an optimization in the attention mechanism that makes inference faster, more efficient and uses less memory.
Expand Down
13 changes: 13 additions & 0 deletions src/evaluator/LlamaContext/utils/resolveContextBatchSizeOption.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import {getDefaultContextBatchSize} from "../LlamaContext.js";
import {padSafeContextSize} from "./padSafeContextSize.js";
import type {LlamaContextOptions} from "../types.js";

export function resolveContextBatchSizeOption({batchSize, contextSize, sequences}: {
batchSize?: LlamaContextOptions["batchSize"],
contextSize: number,
sequences: number
}): number {
return batchSize === "_matchContext"
? padSafeContextSize(contextSize, "up")
: batchSize ?? getDefaultContextBatchSize({contextSize, sequences});
}
18 changes: 16 additions & 2 deletions src/evaluator/LlamaEmbeddingContext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {Token} from "../types.js";
import {LlamaText} from "../utils/LlamaText.js";
import {tokenizeInput} from "../utils/tokenizeInput.js";
import {resolveBeginningTokenToPrepend, resolveEndTokenToAppend} from "../utils/tokenizerUtils.js";
import {LlamaLogLevel} from "../bindings/types.js";
import {LlamaEmbedding} from "./LlamaEmbedding.js";
import type {LlamaModel} from "./LlamaModel/LlamaModel.js";
import type {LlamaContext, LlamaContextSequence} from "./LlamaContext/LlamaContext.js";
Expand All @@ -25,7 +26,11 @@ export type LlamaEmbeddingContextOptions = {
max?: number
},

/** prompt processing batch size */
/**
* Prompt processing batch size.
*
* For non-causal models, always matches the resolved context size regardless of the specified batch size.
*/
batchSize?: number,

/**
Expand Down Expand Up @@ -163,15 +168,24 @@ export class LlamaEmbeddingContext {
if (_model.fileInsights.hasEncoder && _model.fileInsights.hasDecoder)
throw new Error("Computing embeddings is not supported for encoder-decoder models.");

const isNonCausal = _model.fileInfo.architectureMetadata?.attention?.causal === false;
const llamaContext = await _model.createContext({
contextSize,
batchSize,
batchSize: isNonCausal
? "_matchContext"
: batchSize,
threads,
createSignal,
ignoreMemorySafetyChecks,
_embeddings: true
});

if (isNonCausal && batchSize != null && batchSize !== llamaContext.batchSize)
_model._llama._log(LlamaLogLevel.warn,
`Ignoring batchSize ${batchSize} because this non-causal embedding model requires the batch size to match ` +
`the resolved context size (${llamaContext.batchSize}).`
);

return new LlamaEmbeddingContext({
_llamaContext: llamaContext
});
Expand Down
27 changes: 18 additions & 9 deletions src/gguf/insights/GgufInsights.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ import bytes from "bytes";
import {Llama} from "../../bindings/Llama.js";
import {doesLlamaBackendNeedAddonInitLock, LlamaLocks, LlamaLogLevel} from "../../bindings/types.js";
import {getLlamaWithoutBackend} from "../../bindings/utils/getLlamaWithoutBackend.js";
import {getDefaultContextBatchSize, getDefaultContextSequences} from "../../evaluator/LlamaContext/LlamaContext.js";
import {getDefaultContextSequences} from "../../evaluator/LlamaContext/LlamaContext.js";
import {GgufFileInfo} from "../types/GgufFileInfoTypes.js";
import {GgmlType, GgufTensorInfo} from "../types/GgufTensorInfoTypes.js";
import {GgufArchitectureType} from "../types/GgufMetadataTypes.js";
import {getReadablePath} from "../../cli/utils/getReadablePath.js";
import {padSafeContextSize} from "../../evaluator/LlamaContext/utils/padSafeContextSize.js";
import {resolveContextBatchSizeOption} from "../../evaluator/LlamaContext/utils/resolveContextBatchSizeOption.js";
import {removeNullFields, removeUndefinedFields} from "../../utils/removeNullFields.js";
import {LruCache} from "../../utils/LruCache.js";
import {DisposalPreventionHandle, DisposeGuard} from "../../utils/DisposeGuard.js";
Expand Down Expand Up @@ -371,12 +372,16 @@ export class GgufInsights {
swaFullCache = false,
kvCacheKeyType = GgmlType.F16, kvCacheValueType = GgmlType.F16
}: {
contextSize: number, modelGpuLayers: number, batchSize?: number, sequences?: number, isEmbeddingContext?: boolean,
flashAttention?: boolean, includeGraphOverhead?: boolean, swaFullCache?: boolean,
contextSize: number, modelGpuLayers: number, batchSize?: LlamaContextOptions["batchSize"], sequences?: number,
isEmbeddingContext?: boolean, flashAttention?: boolean, includeGraphOverhead?: boolean, swaFullCache?: boolean,
kvCacheKeyType?: GgmlType, kvCacheValueType?: GgmlType
}): GgufInsightsResourceRequirements {
if (sequences == null) sequences = getDefaultContextSequences();
if (batchSize == null) batchSize = getDefaultContextBatchSize({contextSize, sequences});
batchSize = resolveContextBatchSizeOption({
batchSize,
contextSize,
sequences
});

const llmData = this._ggufFileInfo.architectureMetadata;
const tensorInfo = this._ggufFileInfo.fullTensorInfo ?? [];
Expand Down Expand Up @@ -715,8 +720,8 @@ export class GgufInsights {
}

public async estimateContextResourceRequirementsV2(options: {
contextSize: number, modelGpuLayers: number, batchSize?: number, sequences?: number, isEmbeddingContext?: boolean,
flashAttention?: LlamaContextOptions["flashAttention"], swaFullCache?: boolean,
contextSize: number, modelGpuLayers: number, batchSize?: LlamaContextOptions["batchSize"], sequences?: number,
isEmbeddingContext?: boolean, flashAttention?: LlamaContextOptions["flashAttention"], swaFullCache?: boolean,
kvCacheKeyType?: GgmlType, kvCacheValueType?: GgmlType,
useMmap?: boolean,

Expand Down Expand Up @@ -780,13 +785,17 @@ export class GgufInsights {
kvCacheKeyType = GgmlType.F16, kvCacheValueType = GgmlType.F16,
simulatorSession = this._simulationSession
}: {
contextSize: number, modelGpuLayers: number, batchSize?: number, sequences?: number, isEmbeddingContext?: boolean,
flashAttention?: LlamaContextOptions["flashAttention"], swaFullCache?: boolean, useMmap?: boolean,
contextSize: number, modelGpuLayers: number, batchSize?: LlamaContextOptions["batchSize"], sequences?: number,
isEmbeddingContext?: boolean, flashAttention?: LlamaContextOptions["flashAttention"], swaFullCache?: boolean, useMmap?: boolean,
kvCacheKeyType?: GgmlType, kvCacheValueType?: GgmlType,
simulatorSession?: GgufInsightsSimulatorSession
}): Promise<GgufInsightsResourceRequirements | null> {
if (sequences == null) sequences = getDefaultContextSequences();
if (batchSize == null) batchSize = getDefaultContextBatchSize({contextSize, sequences});
batchSize = resolveContextBatchSizeOption({
batchSize,
contextSize,
sequences
});

const kvUnified = false;

Expand Down
5 changes: 5 additions & 0 deletions src/gguf/insights/GgufInsightsConfigurationResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,9 @@ export class GgufInsightsConfigurationResolver {
: useMmap;
let gpuLayersFitMemory = false;
await using simulatorSession = this._ggufInsights._createSimulatorSession();
const batchSize = embeddingContext && this._ggufInsights.ggufFileInfo.architectureMetadata?.attention?.causal === false
? "_matchContext"
: undefined;

try {
const layersResolution = await this.resolveModelGpuLayersV2(
Expand Down Expand Up @@ -266,6 +269,7 @@ export class GgufInsightsConfigurationResolver {

try {
resolvedContextSize = await this.resolveContextContextSize("auto", {
batchSize,
getVramState: async () => ({
total: vramState.total,
free: Math.max(0, vramState.free - estimatedModelResourceUsage.gpuVram),
Expand Down Expand Up @@ -321,6 +325,7 @@ export class GgufInsightsConfigurationResolver {

const estimatedContextResourceUsage = await this._ggufInsights.estimateContextResourceRequirementsV2({
contextSize: resolvedContextSize,
batchSize,
isEmbeddingContext: embeddingContext,
modelGpuLayers: resolvedGpuLayers,
flashAttention,
Expand Down
10 changes: 5 additions & 5 deletions src/gguf/insights/utils/resolveContextContextSizeOption.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import {LlamaContextOptions} from "../../../evaluator/LlamaContext/types.js";
import {GgufInsights, GgufInsightsSimulatorSession} from "../GgufInsights.js";
import {BuildGpu} from "../../../bindings/types.js";
import {minAllowedContextSizeInCalculations} from "../../../config.js";
import {getDefaultContextBatchSize, getDefaultModelContextSize} from "../../../evaluator/LlamaContext/LlamaContext.js";
import {getDefaultModelContextSize} from "../../../evaluator/LlamaContext/LlamaContext.js";
import {InsufficientMemoryError} from "../../../utils/InsufficientMemoryError.js";
import {getRamUsageFromUnifiedVram} from "./getRamUsageFromUnifiedVram.js";
import type {GgmlType} from "../../types/GgufTensorInfoTypes.js";
Expand Down Expand Up @@ -37,7 +37,7 @@ export async function resolveContextContextSizeOption(options: {
kvCacheKeyType, kvCacheValueType, swaFullCache, useMmap = modelFileInsights._llama.supportsMmap,
getVramState, getRamState, getSwapState, ignoreMemorySafetyChecks = false, isEmbeddingContext = false,
maxContextSizeSwapUse = defaultMaxContextSizeSwapUse,

simulatorSession: _simulatorSession,
ramCapIsSet = false,
vramCapIsSet = false
Expand Down Expand Up @@ -67,7 +67,7 @@ export async function resolveContextContextSizeOption(options: {
]);
const contextResourceRequirements = await modelFileInsights.estimateContextResourceRequirementsV2({
contextSize: resolvedContextSize,
batchSize: batchSize ?? getDefaultContextBatchSize({contextSize: resolvedContextSize, sequences}),
batchSize,
modelGpuLayers: modelGpuLayers,
sequences,
flashAttention,
Expand Down Expand Up @@ -118,7 +118,7 @@ export async function resolveContextContextSizeOption(options: {
for (let testContextSize = maxContextSize; testContextSize >= minContextSize && testContextSize <= maxContextSize;) {
const contextResourceRequirements = await modelFileInsights.estimateContextResourceRequirementsV2({
contextSize: testContextSize,
batchSize: batchSize ?? getDefaultContextBatchSize({contextSize: testContextSize, sequences}),
batchSize,
modelGpuLayers: modelGpuLayers,
sequences,
flashAttention,
Expand Down Expand Up @@ -175,7 +175,7 @@ export async function resolveContextContextSizeOption(options: {

const minContextSizeResourceRequirements = await modelFileInsights.estimateContextResourceRequirementsV2({
contextSize: minContextSize,
batchSize: batchSize ?? getDefaultContextBatchSize({contextSize: minContextSize, sequences}),
batchSize,
modelGpuLayers: modelGpuLayers,
sequences,
flashAttention,
Expand Down
8 changes: 6 additions & 2 deletions src/gguf/insights/utils/resolveModelGpuLayersOption.ts
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,9 @@ async function getVramRequiredForGpuLayers({
if (fitContext != null && fitContext.contextSize != null) {
const contextVram = (await ggufInsights.estimateContextResourceRequirementsV2({
contextSize: fitContext.contextSize,
batchSize: getDefaultContextBatchSize({contextSize: fitContext.contextSize, sequences: 1}),
batchSize: fitContext.embeddingContext && ggufInsights.ggufFileInfo.architectureMetadata?.attention?.causal === false
? "_matchContext"
: getDefaultContextBatchSize({contextSize: fitContext.contextSize, sequences: 1}),
modelGpuLayers: gpuLayers,
sequences: 1,
isEmbeddingContext: fitContext.embeddingContext ?? false,
Expand Down Expand Up @@ -476,7 +478,9 @@ async function findMaxPossibleContextSizeForVram({
async test(contextSize) {
const contextVram = (await ggufInsights.estimateContextResourceRequirementsV2({
contextSize,
batchSize: getDefaultContextBatchSize({contextSize, sequences: 1}),
batchSize: isEmbeddingContext && ggufInsights.ggufFileInfo.architectureMetadata?.attention?.causal === false
? "_matchContext"
: getDefaultContextBatchSize({contextSize, sequences: 1}),
modelGpuLayers: gpuLayers,
sequences: 1,
isEmbeddingContext,
Expand Down
8 changes: 8 additions & 0 deletions src/utils/utilTypes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,14 @@ export type Writable<T> = {
-readonly [P in keyof T]: T[P];
};

export type InternalType<T> = {
/**
* Resolves to `T` in source and `never` in built declaration files
* @internal
*/
type: T
} extends {type: infer Type} ? Type : never;

/**
* Omit all the keys from `Value` that are not present in `Options` and are `true`.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1593,8 +1593,8 @@ describe("functionary", () => {
totalRam: s1GB * 8,
freeRam: s1GB * 8
});
expect(res.gpuLayers).to.toMatchInlineSnapshot("0");
expect(res.contextSize).to.toMatchInlineSnapshot("8192");
expect(res.gpuLayers).to.toMatchInlineSnapshot("9");
expect(res.contextSize).to.toMatchInlineSnapshot("7168");
expect(res.useMmap).to.toMatchInlineSnapshot("true");
expect(res.contextSize).to.be.gte(contextSize);
}
Expand Down
7 changes: 4 additions & 3 deletions test/modelDependent/llama3/functions.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -257,9 +257,10 @@ describe("llama 3", () => {

const res = await chatSession.prompt("Is an apple more expensive than a banana?", promptOptions);

expect(res).toMatchInlineSnapshot(
'"Let me check the prices for you. According to the prices I checked, an apple is indeed more expensive than a banana. The apple costs $6, while the banana costs $4."'
);
expect(res).toBeOneOf([
"Let me check the prices for you. According to the prices I checked, an apple is indeed more expensive than a banana. The apple costs $6, while the banana costs $4.",
"Let me check the prices for you. Based on the prices I just checked, it seems that an apple is indeed more expensive than a banana, with an apple costing $6 and a banana costing $4."
]);
});
});

Expand Down
22 changes: 22 additions & 0 deletions test/modelDependent/nomicEmbedText/embedding.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,28 @@ describe("nomic embed text", () => {
expect(helloThereEmbedding2.vector).to.eql(helloThereEmbedding.vector);
});

test("matches the batch size to contextSize 1024 and embeds 600 tokens", {timeout: 1000 * 60 * 60 * 2}, async () => {
const modelPath = await getModelFile("nomic-embed-text-v1.5.Q4_K_M.gguf");
const llama = await getTestLlama();

await using model = await llama.loadModel({modelPath});
expect(model.fileInfo.architectureMetadata.attention?.causal).toBe(false);

await using context = await model.createEmbeddingContext({contextSize: 1024});
expect(context._llamaContext.getAllocatedContextSize()).toBe(1024);
expect(context._llamaContext.batchSize).toBe(1024);

const tokens = model.tokenize("word ".repeat(600));
const specialTokenCount = context.calculateInputLength(tokens) - tokens.length;
const input = tokens.slice(0, 600 - specialTokenCount);
expect(context.calculateInputLength(input)).toBe(600);

const embedding = await context.getEmbeddingFor(input);
expect(embedding.vector).toHaveLength(model.embeddingVectorSize);
expect(embedding.vector.every(Number.isFinite)).toBe(true);
expect(embedding.vector.some((value) => value !== 0)).toBe(true);
});

test("similarity search", {timeout: 1000 * 60 * 60 * 2}, async () => {
const modelPath = await getModelFile("nomic-embed-text-v1.5.Q4_K_M.gguf");
const llama = await getTestLlama();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -619,9 +619,9 @@ describe("stableCode", () => {
totalVram: s1GB * 2,
freeVram: s1GB * 1
});
expect(res.gpuLayers).to.toMatchInlineSnapshot("8");
expect(res.contextSize).to.toMatchInlineSnapshot("6144");
expect(res.useMmap).to.toMatchInlineSnapshot("false");
expect(res.gpuLayers).to.toMatchInlineSnapshot("0");
expect(res.contextSize).to.toMatchInlineSnapshot("16384");
expect(res.useMmap).to.toMatchInlineSnapshot("true");
expect(res.contextSize).to.be.gte(contextSize);
}
{
Expand Down
Loading