Repository navigation
fix: keep text when truncating zero characters in context shift #679
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| import {describe, expect, test} from "vitest"; | ||
| import {GeneralChatWrapper} from "../../../src/chatWrappers/GeneralChatWrapper.js"; | ||
| import { | ||
| eraseFirstResponseAndKeepFirstSystemChatContextShiftStrategy | ||
| } from "../../../src/evaluator/LlamaChat/utils/contextShiftStrategies/eraseFirstResponseAndKeepFirstSystemChatContextShiftStrategy.js"; | ||
| import {ChatHistoryItem, Tokenizer} from "../../../src/types.js"; | ||
|
|
||
|
|
||
| // a fake tokenizer that counts roughly one token per word, so the test doesn't need a model | ||
| const tokenizer = ((text: string) => { | ||
| return Array.from(text.matchAll(/\S+\s*/g)).map((_match, index) => index + 1); | ||
| }) as unknown as Tokenizer; | ||
| (tokenizer as unknown as {detokenize: () => string}).detokenize = () => ""; | ||
|
Comment on lines
+9
to
+13
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think that's going to make maintenance easier, so I'd prefer to either use some real model for this (we can put this test under |
||
|
|
||
| const words = (count: number, prefix: string) => Array.from({length: count}, (_, i) => prefix + i).join(" "); | ||
|
|
||
| describe("eraseFirstResponseAndKeepFirstSystemChatContextShiftStrategy", () => { | ||
| test("does not drop a user message that was meant to be kept entirely", async () => { | ||
| const firstUserMessage = words(60, "ua"); | ||
| const chatHistory: ChatHistoryItem[] = [ | ||
| {type: "system", text: "You are a helpful assistant."}, | ||
| {type: "user", text: firstUserMessage}, | ||
| {type: "user", text: words(60, "ub")}, | ||
| {type: "user", text: words(60, "uc")}, | ||
| {type: "model", response: [words(300, "mc")]} | ||
| ]; | ||
|
|
||
| const res = await eraseFirstResponseAndKeepFirstSystemChatContextShiftStrategy({ | ||
| chatHistory, | ||
| maxTokensCount: 400, | ||
| tokenizer, | ||
| chatWrapper: new GeneralChatWrapper() | ||
| }); | ||
|
|
||
| const userMessages = res.chatHistory.filter((item) => item.type === "user"); | ||
| expect(res.chatHistory[0]).to.eql(chatHistory[0]); | ||
| expect(userMessages.length).toBeGreaterThanOrEqual(2); | ||
| expect(userMessages[0]).to.eql({type: "user", text: firstUserMessage}); | ||
| }); | ||
| }); | ||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,80 @@ | ||||||
| import {describe, expect, test} from "vitest"; | ||||||
| import {truncateLlamaTextAndRoundToWords, truncateTextAndRoundToWords} from "../../../src/utils/truncateTextAndRoundToWords.js"; | ||||||
| import {LlamaText, SpecialToken, SpecialTokensText} from "../../../src/utils/LlamaText.js"; | ||||||
|
|
||||||
|
|
||||||
| describe("utils", () => { | ||||||
| describe("truncateTextAndRoundToWords", () => { | ||||||
| test("truncate size of 0 keeps the whole text", () => { | ||||||
| expect(truncateTextAndRoundToWords("hello world", 0, undefined, false)).toBe("hello world"); | ||||||
| expect(truncateTextAndRoundToWords("hello world", 0, undefined, true)).toBe("hello world"); | ||||||
| expect(truncateTextAndRoundToWords("hello world", 0)).toBe("hello world"); | ||||||
| }); | ||||||
|
|
||||||
| test("negative truncate size keeps the whole text", () => { | ||||||
| expect(truncateTextAndRoundToWords("hello world", -3, undefined, false)).toBe("hello world"); | ||||||
| expect(truncateTextAndRoundToWords("hello world", -3, undefined, true)).toBe("hello world"); | ||||||
| }); | ||||||
|
|
||||||
| test("empty text with truncate size of 0", () => { | ||||||
| expect(truncateTextAndRoundToWords("", 0, undefined, false)).toBe(""); | ||||||
| expect(truncateTextAndRoundToWords("", 0, undefined, true)).toBe(""); | ||||||
| }); | ||||||
|
|
||||||
| test("truncating the entire text results in an empty string", () => { | ||||||
| expect(truncateTextAndRoundToWords("hello world", 11, undefined, false)).toBe(""); | ||||||
| expect(truncateTextAndRoundToWords("hello world", 11, undefined, true)).toBe(""); | ||||||
| expect(truncateTextAndRoundToWords("hello world", 50, undefined, false)).toBe(""); | ||||||
| expect(truncateTextAndRoundToWords("hello world", 50, undefined, true)).toBe(""); | ||||||
| }); | ||||||
|
|
||||||
| test("truncate on a word boundary", () => { | ||||||
| expect(truncateTextAndRoundToWords("hello world", 6, undefined, false)).toBe("hello"); | ||||||
| expect(truncateTextAndRoundToWords("hello world", 6, undefined, true)).toBe("world"); | ||||||
| }); | ||||||
| }); | ||||||
|
|
||||||
| describe("truncateLlamaTextAndRoundToWords", () => { | ||||||
| test("truncate size of 0 or less keeps the whole text", () => { | ||||||
| const text = LlamaText(["abc", new SpecialTokensText("<x>"), "def"]); | ||||||
|
|
||||||
| expect(truncateLlamaTextAndRoundToWords(text, 0, undefined, false).values).to.eql(text.values); | ||||||
| expect(truncateLlamaTextAndRoundToWords(text, 0, undefined, true).values).to.eql(text.values); | ||||||
| expect(truncateLlamaTextAndRoundToWords(text, -3, undefined, false).values).to.eql(text.values); | ||||||
| expect(truncateLlamaTextAndRoundToWords(text, -3, undefined, true).values).to.eql(text.values); | ||||||
| }); | ||||||
|
|
||||||
| test("keeps the remaining items whole when the truncate size is consumed exactly at an item boundary (from end)", () => { | ||||||
| const text = LlamaText(["abc", new SpecialTokensText("<x>"), "def"]); | ||||||
|
|
||||||
| expect(truncateLlamaTextAndRoundToWords(text, 3, undefined, false).values) | ||||||
| .to.eql([...LlamaText(["abc", new SpecialTokensText("<x>")]).values]); | ||||||
| }); | ||||||
|
|
||||||
| test("keeps the remaining items whole when the truncate size is consumed exactly at an item boundary (from start)", () => { | ||||||
| const text = LlamaText(["abc", new SpecialTokensText("<x>"), "def"]); | ||||||
|
|
||||||
| expect(truncateLlamaTextAndRoundToWords(text, 3, undefined, true).values) | ||||||
| .to.eql([...LlamaText([new SpecialTokensText("<x>"), "def"]).values]); | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit:
Suggested change
Also relevant in other places in this file |
||||||
| }); | ||||||
|
|
||||||
| test("keeps a special token that follows an exactly consumed string", () => { | ||||||
| const token = new SpecialToken("BOS"); | ||||||
| const text = LlamaText(["abc", token, "def"]); | ||||||
|
|
||||||
| expect(truncateLlamaTextAndRoundToWords(text, 3, undefined, false).values) | ||||||
| .to.eql([...LlamaText(["abc", token]).values]); | ||||||
| expect(truncateLlamaTextAndRoundToWords(text, 3, undefined, true).values) | ||||||
| .to.eql([...LlamaText([token, "def"]).values]); | ||||||
| }); | ||||||
|
|
||||||
| test("still removes the items that fall within the truncate size", () => { | ||||||
| const text = LlamaText(["abc", new SpecialTokensText("<x>"), "def"]); | ||||||
|
|
||||||
| expect(truncateLlamaTextAndRoundToWords(text, 6, undefined, false).values).to.eql(["abc"]); | ||||||
| expect(truncateLlamaTextAndRoundToWords(text, 6, undefined, true).values).to.eql(["def"]); | ||||||
| expect(truncateLlamaTextAndRoundToWords(text, 100, undefined, false).values).to.eql([]); | ||||||
| expect(truncateLlamaTextAndRoundToWords(text, 100, undefined, true).values).to.eql([]); | ||||||
| }); | ||||||
| }); | ||||||
| }); | ||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Remove these comments