From c4237797972207263f54a986ac2ddc56c09a36d5 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:54:09 -0700 Subject: [PATCH 1/8] fix(copilot): render unclosed special tags as text on completed messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The special-tag parser suppressed everything after an opening tag with no close, even after streaming ended — so an assistant message that merely MENTIONED `` in prose lost its entire remainder in the UI. The stream itself completed fine; only the render truncated. Suppression now applies only while streaming. A completed message can never finish an unclosed tag, so the marker was literal text and the remainder is rendered as-is. Originally written alongside the docs/ VFS work and reverted at review time to keep that PR to one concern; this restores it on its own. --- .../components/special-tags/special-tags.test.ts | 15 +++++++++++++++ .../components/special-tags/special-tags.tsx | 14 ++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts index 09444ecccbf..d6a93c3ee8c 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts @@ -149,6 +149,21 @@ describe('parseSpecialTags with ', () => { expect(segments).toEqual([{ type: 'text', content: 'Thinking about it. ' }]) }) + it('renders an unclosed tag as text once the message is complete', () => { + const content = + 'The `` file chip only renders when its path points to a real file.' + const { segments, hasPendingTag } = parseSpecialTags(content, false) + expect(hasPendingTag).toBe(false) + expect(segments).toEqual([ + { type: 'text', content: 'The `' }, + { + type: 'text', + content: + '` file chip only renders when its path points to a real file.', + }, + ]) + }) + it('strips a trailing partial opening tag while streaming', () => { const { segments, hasPendingTag } = parseSpecialTags('Let me ask. ` + // chip"), not a real tag. Render the remainder instead of swallowing it. + const remaining = content.slice(nearestStart) + if (remaining.trim()) { + segments.push({ type: 'text', content: remaining }) + } break } From 6f4896b6bba6bb0fba667a6cfc99daa216483070 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:23:27 -0700 Subject: [PATCH 2/8] improvement(copilot): show text as soon as an unclosed tag cannot resolve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restoring the text only at end of stream left a real gap: once the model mentions a tag in prose, everything after it stays invisible for the rest of the stream and reappears in one jump when streaming stops. On a long reply that is most of the message. Two properties let us decide much earlier, both conservative — they only fire on content that could not have parsed: - Tags never nest, so any special-tag marker inside the body (opening OR closing, not just a mismatched close) proves the opener was literal text. - JSON-bodied tags must start with `{` or `[`, so the first non-space character settles it — prose after the marker is caught on the very next chunk rather than at the end. The second is per-tag, not global: `thinking` bodies are prose by design (parseTextTagBody), so the JSON rule cannot apply there and only the nesting rule can rescue it. A test documents that remaining gap rather than leaving it implicit. A false positive is cheap by construction: text shows early and the end-of-stream parse still produces the correct final render. --- .../special-tags/special-tags.test.ts | 41 ++++++++++++ .../components/special-tags/special-tags.tsx | 63 +++++++++++++++++-- 2 files changed, 100 insertions(+), 4 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts index d6a93c3ee8c..cf79ecbc375 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts @@ -149,6 +149,47 @@ describe('parseSpecialTags with ', () => { expect(segments).toEqual([{ type: 'text', content: 'Thinking about it. ' }]) }) + it('shows prose immediately mid-stream instead of blanking the rest', () => { + // The failure this replaces: everything after the marker stayed invisible + // for the remainder of the stream, then reappeared when it ended. + const content = 'The `` chip only renders for a real file.' + const { segments, hasPendingTag } = parseSpecialTags(content, true) + expect(hasPendingTag).toBe(false) + expect(segments.map((s) => ('content' in s ? s.content : s.type)).join('')).toContain( + 'chip only renders for a real file.' + ) + }) + + it('still suppresses a JSON-bodied tag that is genuinely mid-stream', () => { + const { segments, hasPendingTag } = parseSpecialTags( + 'Here you go {"type":"file","id":"abc"', + true + ) + expect(hasPendingTag).toBe(true) + expect(segments).toEqual([{ type: 'text', content: 'Here you go ' }]) + }) + + it('bails when a foreign closing tag appears inside the body', () => { + // Tags never nest, so a close for a different tag proves the opener was text. + const { hasPendingTag } = parseSpecialTags( + 'see [{"title":"a","description":"b"}] more', + true + ) + expect(hasPendingTag).toBe(false) + }) + + it('bails on a nested opening tag', () => { + const { hasPendingTag } = parseSpecialTags('a b c', true) + expect(hasPendingTag).toBe(false) + }) + + it('keeps suppressing an unclosed thinking tag with prose — its body is not JSON', () => { + // Documents the deliberate gap: `thinking` bodies are prose, so the JSON + // heuristic cannot apply and only the nesting rule can rescue it. + const { hasPendingTag } = parseSpecialTags('a still reasoning about', true) + expect(hasPendingTag).toBe(true) + }) + it('renders an unclosed tag as text once the message is complete', () => { const content = 'The `` file chip only renders when its path points to a real file.' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 6906561ff8f..3355cb10327 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -473,6 +473,56 @@ function parseSpecialTagData( * Trailing partial opening tags (e.g. `([ + 'options', + 'usage_upgrade', + 'credential', + 'mothership-error', + 'workspace_resource', + 'question', +]) + +/** + * True when an opening tag with no close yet can NEVER resolve, so the text + * after it should be shown immediately instead of held back until the stream + * ends. + * + * Without this, a message that merely mentions a tag in prose goes blank from + * that point on for the rest of the stream — the text is only restored once + * streaming stops. Two facts let us decide early: + * + * 1. Tags never nest. Any other special-tag marker inside the body — opening or + * closing — means this opener was literal text, not a real tag. + * 2. JSON-bodied tags must start with `{` or `[`. The first non-space character + * settles it, so prose after the marker is caught on the very next chunk. + * + * Both are conservative: they only fire on content that could not have parsed. + * A false positive would merely show text early that a later chunk resolves + * into a tag — the end-of-stream parse still produces the correct final render. + */ +function unclosedTagCannotResolve( + tagName: (typeof SPECIAL_TAG_NAMES)[number], + body: string +): boolean { + for (const name of SPECIAL_TAG_NAMES) { + // A close for this tag is absent by definition here, so this catches a + // FOREIGN close; the open check catches nesting, including self-nesting. + if (body.includes(``) || body.includes(`<${name}>`)) return true + } + + if (JSON_BODY_TAG_NAMES.has(tagName)) { + const firstChar = body.trimStart().charAt(0) + if (firstChar !== '' && firstChar !== '{' && firstChar !== '[') return true + } + + return false +} + export function parseSpecialTags(content: string, isStreaming: boolean): ParsedSpecialContent { const segments: ContentSegment[] = [] let hasPendingTag = false @@ -526,14 +576,19 @@ export function parseSpecialTags(content: string, isStreaming: boolean): ParsedS const closeIdx = content.indexOf(closeTag, bodyStart) if (closeIdx === -1) { - if (isStreaming) { + // Hold the text back only while a close is still plausible. A completed + // message can never finish an unclosed tag, and mid-stream the heuristics + // in unclosedTagCannotResolve rule it out early — otherwise a tag merely + // mentioned in prose blanks the rest of the message until the stream ends. + const stillResolvable = + isStreaming && + nearestTagName !== '' && + !unclosedTagCannotResolve(nearestTagName, content.slice(bodyStart)) + if (stillResolvable) { hasPendingTag = true cursor = content.length break } - // A completed message can never finish an unclosed tag — the marker was - // literal text mentioning the tag (e.g. "the `` - // chip"), not a real tag. Render the remainder instead of swallowing it. const remaining = content.slice(nearestStart) if (remaining.trim()) { segments.push({ type: 'text', content: remaining }) From f0ff34b20369f9a106b0e948998dab49fd58465e Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:29:40 -0700 Subject: [PATCH 3/8] fix(copilot): ignore tag syntax quoted inside a JSON tag body MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nesting rule treated any tag marker in the body as proof the opener was literal text. But a JSON body can legitimately quote tag syntax — a `` asking which tag to use, a `` whose title mentions one — and that is exactly the "model explains its own tags" situation this whole fix exists for. Bailing there is the one expensive false positive in the design: the raw JSON renders and STAYS for the rest of the stream, then snaps into a card when the real close arrives. More jarring than the bug being fixed. For JSON-bodied tags the nesting scan now runs over a copy with string literals blanked (escape-aware, and tolerant of the unterminated trailing string that is normal mid-stream). Markers in real body position still count. `thinking` is unaffected — its body is prose, so there are no strings to confuse it. Tests cover both halves: the streaming case does not bail, and the same body resolves to a question card once it closes. --- .../special-tags/special-tags.test.ts | 23 ++++++++++ .../components/special-tags/special-tags.tsx | 45 ++++++++++++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts index cf79ecbc375..80724d94b09 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts @@ -178,6 +178,29 @@ describe('parseSpecialTags with ', () => { expect(hasPendingTag).toBe(false) }) + it('does not bail on tag syntax quoted inside a JSON string', () => { + // The false positive this guards: a question whose text legitimately quotes + // another tag. Bailing would show raw JSON that later snaps into a card. + const streaming = 'ok [{"type":"single_select","prompt":"Use the tag?"' + expect(parseSpecialTags(streaming, true).hasPendingTag).toBe(true) + }) + + it('resolves that same question correctly once it closes', () => { + // The other half of the guarantee: the body the streaming case refused to + // bail on does render as a question card, so nothing flickered for nothing. + const complete = + 'ok [{"type":"single_select","prompt":"Use the tag?","options":[{"id":"y","label":"Yes"},{"id":"n","label":"No"}]}]' + const { segments } = parseSpecialTags(complete, false) + expect(segments.some((s) => s.type === 'question')).toBe(true) + }) + + it('still bails on a marker outside the JSON strings', () => { + // Escapes must not end the string early, and a marker in real body position + // is still evidence. + const streaming = 'ok [{"prompt":"a \\" quote"} ' + expect(parseSpecialTags(streaming, true).hasPendingTag).toBe(false) + }) + it('bails on a nested opening tag', () => { const { hasPendingTag } = parseSpecialTags('a b c', true) expect(hasPendingTag).toBe(false) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 3355cb10327..6b1863f51ff 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -487,6 +487,44 @@ const JSON_BODY_TAG_NAMES = new Set<(typeof SPECIAL_TAG_NAMES)[number]>([ 'question', ]) +/** + * Strip the contents of JSON string literals from `body`, replacing them with + * spaces so every other index is preserved. + * + * A JSON tag body can legitimately quote tag syntax — a `` asking + * which tag to use, or a `` whose title mentions one. Those + * markers live inside a string and say nothing about whether the tag will + * close, so the nesting rule must not see them. Tracks escapes so a `\"` inside + * a string does not end it early. Handles an unterminated trailing string, which + * is the normal state mid-stream. + */ +function blankJsonStringLiterals(body: string): string { + let out = '' + let inString = false + let escaped = false + + for (const char of body) { + if (escaped) { + escaped = false + out += ' ' + continue + } + if (char === '\\' && inString) { + escaped = true + out += ' ' + continue + } + if (char === '"') { + inString = !inString + out += '"' + continue + } + out += inString ? ' ' : char + } + + return out +} + /** * True when an opening tag with no close yet can NEVER resolve, so the text * after it should be shown immediately instead of held back until the stream @@ -509,10 +547,15 @@ function unclosedTagCannotResolve( tagName: (typeof SPECIAL_TAG_NAMES)[number], body: string ): boolean { + // For a JSON-bodied tag, ignore markers inside string literals: quoting tag + // syntax is legitimate content, and treating it as evidence would bail on a + // tag that goes on to close correctly — showing raw JSON that then snaps into + // a rendered card. + const scannable = JSON_BODY_TAG_NAMES.has(tagName) ? blankJsonStringLiterals(body) : body for (const name of SPECIAL_TAG_NAMES) { // A close for this tag is absent by definition here, so this catches a // FOREIGN close; the open check catches nesting, including self-nesting. - if (body.includes(``) || body.includes(`<${name}>`)) return true + if (scannable.includes(``) || scannable.includes(`<${name}>`)) return true } if (JSON_BODY_TAG_NAMES.has(tagName)) { From ba545fa2e24c841a21f7a6db6024db0d4d4ba915 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:16:48 -0700 Subject: [PATCH 4/8] fix(copilot): keep prose a mispaired tag would swallow, and bail on dead JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more real cases from traces, both of which the earlier heuristics missed. 1. A MATCHED pair whose body fails to parse was silently dropped, cursor and all. When the model explains tag syntax and ends with a backticked example containing a real closing tag, that example closes an EARLIER opener and three paragraphs become the "body" — which is not valid JSON, so the whole span vanished and the render resumed mid-sentence (trace b095e080). Now emitted verbatim, but only when the body contains a tag-shaped marker, which is what shows the pairing was wrong. A marker-free body that merely fails validation is a genuinely malformed agent payload and keeps being dropped — an existing test asserts that deliberately, and showing the user raw JSON there would be a regression. 2. The JSON heuristic only checked the FIRST character, so `{"type":"file"}', () => { expect(segments).toEqual([{ type: 'text', content: 'Thinking about it. ' }]) }) + it('keeps the text when a matched pair fails to parse', () => { + // Verbatim from a real message (trace b095e080). The model explained the + // tag and ended with a backticked example containing a REAL closing tag, + // which closed the earlier opener and made everything between it the body. + // That body is not valid JSON, so the segment was dropped and the render + // resumed mid-sentence at ") is what actually produces the interactive + // chip." — three paragraphs silently gone. + const raw = + 'Here you go — with the ending tag intentionally malformed as ``:\n\n' + + '{"type": "file", "path": "files/notes.md", "title": "notes.md"}\n\n' + + "Since the closing tag doesn't match the opening ``, the chat won't " + + 'recognize it as a valid resource chip. A properly matched pair ' + + '(`...`) is what actually produces the interactive chip.' + + const rendered = parseSpecialTags(raw, false) + .segments.map((segment) => ('content' in segment ? segment.content : '')) + .join('') + + expect(rendered).toContain("Since the closing tag doesn't match") + expect(rendered).toContain('A properly matched pair') + expect(rendered).toContain('"path": "files/notes.md"') + // No segment renders as a resource chip — the body was never valid. + expect(parseSpecialTags(raw, false).segments.some((s) => s.type === 'workspace_resource')).toBe( + false + ) + }) + + it('still drops a marker-free malformed payload rather than showing raw JSON', () => { + // The complement of the case above: no tag markers in the body, so this is + // a genuinely broken emission from the agent, not swallowed prose. + const { segments } = parseSpecialTags( + 'Before. {"type":"single_select"} After.', + false + ) + expect(segments).toEqual([ + { type: 'text', content: 'Before. ' }, + { type: 'text', content: ' After.' }, + ]) + }) + + it('still renders a matched pair whose body IS valid', () => { + const raw = + 'see {"type":"file","path":"files/a.md","title":"a.md"} ok' + const { segments } = parseSpecialTags(raw, false) + expect(segments.some((s) => s.type === 'workspace_resource')).toBe(true) + }) + it('shows prose immediately mid-stream instead of blanking the rest', () => { // The failure this replaces: everything after the marker stayed invisible // for the remainder of the stream, then reappeared when it ended. @@ -160,6 +207,24 @@ describe('parseSpecialTags with ', () => { ) }) + it('shows text once the JSON value has closed and stray content follows', () => { + // Verbatim shape from a real message (trace afbeefd0): the close tag was + // TRUNCATED to `{"type":"file","path":"files/notes.md"} ('content' in s ? s.content : '')).join('')).toContain( + 'I brew a cup of coffee' + ) + }) + + it('tolerates braces inside JSON strings when tracking depth', () => { + const raw = 'x {"title":"a } b","path":"files/a.md"' + expect(parseSpecialTags(raw, true).hasPendingTag).toBe(true) + }) + it('still suppresses a JSON-bodied tag that is genuinely mid-stream', () => { const { segments, hasPendingTag } = parseSpecialTags( 'Here you go {"type":"file","id":"abc"', diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 6b1863f51ff..63dbe48a576 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -473,6 +473,12 @@ function parseSpecialTagData( * Trailing partial opening tags (e.g. `` is exactly the case that matters. + */ +const TAG_SHAPED_MARKER = /<\/?[a-zA-Z][\w-]*>/ + /** * Tags whose body must be JSON. `thinking` is the exception — its body is prose * (see {@link parseTextTagBody}), so a non-JSON body there says nothing about @@ -525,6 +531,41 @@ function blankJsonStringLiterals(body: string): string { return out } +/** + * True while `body` could still grow into a single valid JSON value. + * + * Checking only the first character is not enough: a body like + * `{"type":"file"}`) || scannable.includes(`<${name}>`)) return true } - if (JSON_BODY_TAG_NAMES.has(tagName)) { - const firstChar = body.trimStart().charAt(0) - if (firstChar !== '' && firstChar !== '{' && firstChar !== '[') return true - } + if (JSON_BODY_TAG_NAMES.has(tagName) && !isViableJsonPrefix(body)) return true return false } @@ -647,6 +685,21 @@ export function parseSpecialTags(content: string, isStreaming: boolean): ParsedS const parsedTag = parseSpecialTagData(nearestTagName, body) if (parsedTag) { segments.push(parsedTag) + } else if (TAG_SHAPED_MARKER.test(body)) { + // The body failed to parse AND contains tag markers, which means the + // close we matched was not this opener's — the model was explaining tag + // syntax and a later example like `...` + // closed an earlier opener, making paragraphs of prose the "body". + // Dropping that silently loses the text and resumes mid-sentence, so emit + // it verbatim. Streamdown escapes the markers, so it reads as literal text. + // + // A marker-free body that merely fails validation is a genuinely + // malformed payload from the agent; that keeps being dropped rather than + // showing the user raw JSON. + const literal = content.slice(nearestStart, closeIdx + closeTag.length) + if (literal.trim()) { + segments.push({ type: 'text', content: literal }) + } } cursor = closeIdx + closeTag.length From 163fadacb4cb1ff26963f8a284a71b7f181d4a8f Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:24:47 -0700 Subject: [PATCH 5/8] fix(copilot): keep prose a tag wrapped instead of a JSON payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third real case from a trace (1206fd8a): `the gmail-agent workflow` — a matched pair whose body is plain prose. The sentence rendered as "...once I wired up to handle the welcome sequence" with its subject silently removed. The marker test could not fire (prose contains no tag markers), so it fell to the deliberate drop-malformed-payload path. But that path exists for an agent emitting BROKEN JSON, not for a tag wrapping prose. The distinction is whether the body was ever an attempted payload, which isViableJsonPrefix already answers: `{"type":"single_select"}` is a well-formed JSON value failing its shape guard and keeps being dropped; `the gmail-agent workflow` was never a payload and is emitted. --- .../special-tags/special-tags.test.ts | 13 ++++++++++ .../components/special-tags/special-tags.tsx | 24 +++++++++++-------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts index 5024bc89de7..aa87d2aba86 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts @@ -176,6 +176,19 @@ describe('parseSpecialTags with ', () => { ) }) + it('keeps prose a tag wrapped instead of a payload', () => { + // Verbatim from a real message (trace 1206fd8a): a matched pair whose body + // is plain prose, never an attempted JSON payload. The sentence read + // "...once I wired up to handle the welcome sequence" with the subject gone. + const raw = + 'once I wired up the gmail-agent workflow to handle the welcome sequence.' + const rendered = parseSpecialTags(raw, false) + .segments.map((segment) => ('content' in segment ? segment.content : '')) + .join('') + expect(rendered).toContain('the gmail-agent workflow') + expect(rendered).toContain('to handle the welcome sequence') + }) + it('still drops a marker-free malformed payload rather than showing raw JSON', () => { // The complement of the case above: no tag markers in the body, so this is // a genuinely broken emission from the agent, not swallowed prose. diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 63dbe48a576..0dc8c52050d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -685,17 +685,21 @@ export function parseSpecialTags(content: string, isStreaming: boolean): ParsedS const parsedTag = parseSpecialTagData(nearestTagName, body) if (parsedTag) { segments.push(parsedTag) - } else if (TAG_SHAPED_MARKER.test(body)) { - // The body failed to parse AND contains tag markers, which means the - // close we matched was not this opener's — the model was explaining tag - // syntax and a later example like `...` - // closed an earlier opener, making paragraphs of prose the "body". - // Dropping that silently loses the text and resumes mid-sentence, so emit - // it verbatim. Streamdown escapes the markers, so it reads as literal text. + } else if ( + // The close we matched was not this opener's: the model was explaining tag + // syntax and a later example closed an earlier opener, making paragraphs + // of prose the "body". + TAG_SHAPED_MARKER.test(body) || + // Or the tag wrapped prose that was never an attempted payload at all. + (JSON_BODY_TAG_NAMES.has(nearestTagName) && !isViableJsonPrefix(body)) + ) { + // Either way the markers were literal text, so dropping the span loses + // real content and resumes mid-sentence. Emit it verbatim — Streamdown + // escapes the markers, so it reads as literal text. // - // A marker-free body that merely fails validation is a genuinely - // malformed payload from the agent; that keeps being dropped rather than - // showing the user raw JSON. + // A body that IS a well-formed JSON value and merely fails its shape + // guard keeps being dropped: that is a genuinely malformed payload from + // the agent, and showing the user raw JSON there would be a regression. const literal = content.slice(nearestStart, closeIdx + closeTag.length) if (literal.trim()) { segments.push({ type: 'text', content: literal }) From 1f5918ce51ea4e49d4f137509680c28834ecf5b4 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:33:51 -0700 Subject: [PATCH 6/8] refactor(copilot): resolve each special tag through four named outcomes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parseSpecialTags had grown five inline branches, each added for a specific malformation found in a trace. The shape made "drop it" the implicit fallback, which is how spans that were never malformed payloads ended up silently swallowed. Extracts resolveTagAt, returning one of four named outcomes — segment, literal, discard, pending — so each decision is explicit and the main loop just dispatches on it. Fixes a latent bug the old shape hid: rejecting an unclosed tag ran `break`, abandoning the rest of the message, so a genuinely valid tag after a literal mention was never parsed. Resolution now resumes just past the rejected opener and scanning continues. Test added. Two behavior notes: - Rejected spans are emitted in smaller pieces. The renderer concatenates adjacent text segments into one markdown string, so this is display-neutral; the two tests that asserted exact segment arrays now assert joined text. - Each opener is judged on its own evidence. Previously one verdict ended the whole parse, so a nested opener released everything; now the outer is released immediately and the inner is a fresh candidate that can still hold mid-stream. It resolves at end of stream either way. --- .../special-tags/special-tags.test.ts | 44 ++++-- .../components/special-tags/special-tags.tsx | 146 ++++++++++-------- 2 files changed, 118 insertions(+), 72 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts index aa87d2aba86..b45ebcce802 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts @@ -176,6 +176,16 @@ describe('parseSpecialTags with ', () => { ) }) + it('still parses a valid tag that follows a rejected one', () => { + // Before the rewrite, rejecting an unclosed tag abandoned the rest of the + // message, so this tag was never parsed at all. + const { segments } = parseSpecialTags( + 'I use loosely here. Anyway: [{"title":"A","description":"d"}] done.', + false + ) + expect(segments.map((segment) => segment.type)).toContain('options') + }) + it('keeps prose a tag wrapped instead of a payload', () => { // Verbatim from a real message (trace 1206fd8a): a matched pair whose body // is plain prose, never an attempted JSON payload. The sentence read @@ -279,9 +289,22 @@ describe('parseSpecialTags with ', () => { expect(parseSpecialTags(streaming, true).hasPendingTag).toBe(false) }) - it('bails on a nested opening tag', () => { - const { hasPendingTag } = parseSpecialTags('a b c', true) - expect(hasPendingTag).toBe(false) + it('rejects an opener a nested one disproves, then judges the inner on its own', () => { + // Each opener is evaluated independently. The first is disproved by the + // nested opener and its text is released immediately; the second is a fresh + // candidate that nothing has ruled out yet, so it holds mid-stream. + const streaming = parseSpecialTags('a b c', true) + expect(streaming.hasPendingTag).toBe(true) + expect( + streaming.segments.map((segment) => ('content' in segment ? segment.content : '')).join('') + ).toBe('a b ') + + // Once the stream ends nothing can close it, so the whole line is shown. + const done = parseSpecialTags('a b c', false) + expect(done.hasPendingTag).toBe(false) + expect( + done.segments.map((segment) => ('content' in segment ? segment.content : '')).join('') + ).toBe('a b c') }) it('keeps suppressing an unclosed thinking tag with prose — its body is not JSON', () => { @@ -296,14 +319,13 @@ describe('parseSpecialTags with ', () => { 'The `` file chip only renders when its path points to a real file.' const { segments, hasPendingTag } = parseSpecialTags(content, false) expect(hasPendingTag).toBe(false) - expect(segments).toEqual([ - { type: 'text', content: 'The `' }, - { - type: 'text', - content: - '` file chip only renders when its path points to a real file.', - }, - ]) + // Asserted on the joined text, not segment boundaries: the renderer + // concatenates adjacent text segments, so how the span is split is not + // observable to a reader. + expect(segments.every((segment) => segment.type === 'text')).toBe(true) + expect(segments.map((segment) => ('content' in segment ? segment.content : '')).join('')).toBe( + content + ) }) it('strips a trailing partial opening tag while streaming', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 0dc8c52050d..1c39daaa263 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -604,11 +604,83 @@ function unclosedTagCannotResolve( return false } +/** + * How one opening tag resolved. Naming the four outcomes is the point: the + * parser previously decided each case inline, which is how "drop it" quietly + * became the fallback for situations that were never malformed payloads. + */ +type TagResolution = + /** Body parsed; emit the typed segment and resume after the closing tag. */ + | { outcome: 'segment'; segment: ContentSegment; resumeAt: number } + /** Provably not a tag; render the span verbatim and resume after it. */ + | { outcome: 'literal'; resumeAt: number } + /** A well-formed payload that failed its shape guard — dropped deliberately. */ + | { outcome: 'discard'; resumeAt: number } + /** Still streaming and a close remains plausible; suppress the remainder. */ + | { outcome: 'pending' } + +/** + * True when a failed body was never an attempted payload — so the markers were + * literal text and the span must be shown rather than swallowed. + * + * Either the close we matched belongs to a different opener (the body carries + * tag markers), or the tag wrapped prose that was never JSON to begin with. + */ +function bodyIsLiteralText(tagName: (typeof SPECIAL_TAG_NAMES)[number], body: string): boolean { + if (TAG_SHAPED_MARKER.test(body)) return true + return JSON_BODY_TAG_NAMES.has(tagName) && !isViableJsonPrefix(body) +} + +function resolveTagAt( + content: string, + openIndex: number, + tagName: (typeof SPECIAL_TAG_NAMES)[number], + isStreaming: boolean +): TagResolution { + const openTag = `<${tagName}>` + const closeTag = `` + const bodyStart = openIndex + openTag.length + const closeIdx = content.indexOf(closeTag, bodyStart) + + if (closeIdx === -1) { + if (isStreaming && !unclosedTagCannotResolve(tagName, content.slice(bodyStart))) { + return { outcome: 'pending' } + } + // Nothing can close it, so only the opener itself is literal. Resuming just + // past it (rather than abandoning the message) keeps a genuinely valid tag + // later in the same reply parseable. + return { outcome: 'literal', resumeAt: bodyStart } + } + + const resumeAt = closeIdx + closeTag.length + const body = content.slice(bodyStart, closeIdx) + + const parsed = parseSpecialTagData(tagName, body) + if (parsed) return { outcome: 'segment', segment: parsed, resumeAt } + + if (bodyIsLiteralText(tagName, body)) return { outcome: 'literal', resumeAt } + + // A well-formed value that failed its shape guard is a broken emission from + // the agent; showing the user raw JSON there would be worse than nothing. + return { outcome: 'discard', resumeAt } +} + +/** + * Splits streamed text into renderable segments, extracting complete special + * tags and deciding what to do with the ones that never resolve. + * + * Adjacent text segments are concatenated by the renderer, so emitting a span + * as several pieces is display-neutral. + */ export function parseSpecialTags(content: string, isStreaming: boolean): ParsedSpecialContent { const segments: ContentSegment[] = [] let hasPendingTag = false let cursor = 0 + const pushText = (text: string) => { + if (text.trim()) segments.push({ type: 'text', content: text }) + } + while (cursor < content.length) { let nearestStart = -1 let nearestTagName: (typeof SPECIAL_TAG_NAMES)[number] | '' = '' @@ -621,10 +693,11 @@ export function parseSpecialTags(content: string, isStreaming: boolean): ParsedS } } - if (nearestStart === -1) { + if (nearestStart === -1 || nearestTagName === '') { let remaining = content.slice(cursor) if (isStreaming) { + // Hide a half-arrived opening marker so it does not flash as text. const partial = remaining.match(/<[a-z_-]*$/i) if (partial) { const fragment = partial[0].slice(1) @@ -638,75 +711,26 @@ export function parseSpecialTags(content: string, isStreaming: boolean): ParsedS } } - if (remaining.trim()) { - segments.push({ type: 'text', content: remaining }) - } + pushText(remaining) break } - if (nearestStart > cursor) { - const text = content.slice(cursor, nearestStart) - if (text.trim()) { - segments.push({ type: 'text', content: text }) - } - } + pushText(content.slice(cursor, nearestStart)) - const openTag = `<${nearestTagName}>` - const closeTag = `` - const bodyStart = nearestStart + openTag.length - const closeIdx = content.indexOf(closeTag, bodyStart) - - if (closeIdx === -1) { - // Hold the text back only while a close is still plausible. A completed - // message can never finish an unclosed tag, and mid-stream the heuristics - // in unclosedTagCannotResolve rule it out early — otherwise a tag merely - // mentioned in prose blanks the rest of the message until the stream ends. - const stillResolvable = - isStreaming && - nearestTagName !== '' && - !unclosedTagCannotResolve(nearestTagName, content.slice(bodyStart)) - if (stillResolvable) { - hasPendingTag = true - cursor = content.length - break - } - const remaining = content.slice(nearestStart) - if (remaining.trim()) { - segments.push({ type: 'text', content: remaining }) - } + const resolution = resolveTagAt(content, nearestStart, nearestTagName, isStreaming) + + if (resolution.outcome === 'pending') { + hasPendingTag = true break } - const body = content.slice(bodyStart, closeIdx) - if (!nearestTagName) { - cursor = closeIdx + closeTag.length - continue - } - const parsedTag = parseSpecialTagData(nearestTagName, body) - if (parsedTag) { - segments.push(parsedTag) - } else if ( - // The close we matched was not this opener's: the model was explaining tag - // syntax and a later example closed an earlier opener, making paragraphs - // of prose the "body". - TAG_SHAPED_MARKER.test(body) || - // Or the tag wrapped prose that was never an attempted payload at all. - (JSON_BODY_TAG_NAMES.has(nearestTagName) && !isViableJsonPrefix(body)) - ) { - // Either way the markers were literal text, so dropping the span loses - // real content and resumes mid-sentence. Emit it verbatim — Streamdown - // escapes the markers, so it reads as literal text. - // - // A body that IS a well-formed JSON value and merely fails its shape - // guard keeps being dropped: that is a genuinely malformed payload from - // the agent, and showing the user raw JSON there would be a regression. - const literal = content.slice(nearestStart, closeIdx + closeTag.length) - if (literal.trim()) { - segments.push({ type: 'text', content: literal }) - } + if (resolution.outcome === 'segment') { + segments.push(resolution.segment) + } else if (resolution.outcome === 'literal') { + pushText(content.slice(nearestStart, resolution.resumeAt)) } - cursor = closeIdx + closeTag.length + cursor = resolution.resumeAt } if (segments.length === 0 && !hasPendingTag) { From ef6cc84c8c6afc4cb831977105706a27915e47b7 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:54:48 -0700 Subject: [PATCH 7/8] test(copilot): pin the no-closing-tag case as lossless MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourth malformation from a trace (220cc02d): the model wrote an opening tag with valid JSON and no closing tag of any kind. No marker rule can fire — there is no marker — but the JSON value completes and prose follows, which the depth rule settles at the first space after the `}`. Already handled by that rule; this pins it. Asserted as lossless rather than merely visible: mid-stream and complete, every character of the message survives, so nothing waits for the stream to end. --- .../components/special-tags/special-tags.test.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts index b45ebcce802..b1d7d84f920 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts @@ -186,6 +186,22 @@ describe('parseSpecialTags with ', () => { expect(segments.map((segment) => segment.type)).toContain('options') }) + it('loses nothing when the model writes no closing tag at all', () => { + // Verbatim from a real message (trace 220cc02d). No close tag exists, so no + // marker rule can fire — but the JSON value completes and prose follows, + // which settles it at the first space. Asserted as LOSSLESS: mid-stream and + // complete, every character survives. + const raw = + 'The dataset lives in {"type": "file", "path": "files/notes.md"} and I keep coming back to it whenever I need a quick reference. It never quite has everything.' + const join = (result: ReturnType) => + result.segments.map((segment) => ('content' in segment ? segment.content : '')).join('') + + const streaming = parseSpecialTags(raw, true) + expect(streaming.hasPendingTag).toBe(false) + expect(join(streaming)).toBe(raw) + expect(join(parseSpecialTags(raw, false))).toBe(raw) + }) + it('keeps prose a tag wrapped instead of a payload', () => { // Verbatim from a real message (trace 1206fd8a): a matched pair whose body // is plain prose, never an attempted JSON payload. The sentence read From c61dd9dc73f7fd25ac94a42f234f974ee0364b03 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Fri, 24 Jul 2026 23:58:21 -0700 Subject: [PATCH 8/8] fix(copilot): stop the literal path flashing payloads and rescanning the buffer Review round on the four-outcome rewrite. Three defects in how an opener is judged, plus the cost of judging it. A valid tag showed its raw payload as text while its closing marker streamed in. The JSON value closes at the `}`, so a half-arrived ` block. dropArrivingClose now ignores a trailing fragment that could still grow into this tag's own close. Evidence that a close is genuinely wrong still lands immediately: a misspelled `` is not a prefix of ``, and a truncated `', () => { it('still drops a marker-free malformed payload rather than showing raw JSON', () => { // The complement of the case above: no tag markers in the body, so this is // a genuinely broken emission from the agent, not swallowed prose. - const { segments } = parseSpecialTags( + const { segments, hasPendingTag } = parseSpecialTags( 'Before. {"type":"single_select"} After.', false ) + expect(hasPendingTag).toBe(false) expect(segments).toEqual([ { type: 'text', content: 'Before. ' }, { type: 'text', content: ' After.' }, ]) }) + it('drops that same payload even when its JSON quotes tag syntax', () => { + // The marker scan must blank JSON strings the way the streaming path does. + // Scanning the raw body sees `` inside the prompt, calls the span + // literal text, and renders the raw payload — the outcome `discard` exists + // to prevent. + const { segments } = parseSpecialTags( + 'A [{"type":"single_select","prompt":"use here?"}] B', + false + ) + expect(segments).toEqual([ + { type: 'text', content: 'A ' }, + { type: 'text', content: ' B' }, + ]) + }) + + it('does not flash the payload while the closing tag is still arriving', () => { + // Each frame below is a real mid-stream state: the JSON value has closed, so + // without tolerating an arriving close the trailing ``. + for (const fragment of ['<', '[{"title":"a","description":"b"}]${fragment}`, + true + ) + expect(hasPendingTag).toBe(true) + expect(segments.map((s) => ('content' in s ? s.content : '')).join('')).toBe('see ') + } + }) + + it('still rejects a close whose name is wrong rather than merely unfinished', () => { + // The counterpart to the case above: `` can never grow + // into ``, so it settles immediately instead of hiding + // the rest of the message for the remainder of the stream. + const { hasPendingTag } = parseSpecialTags( + 'see {"type":"file","path":"a.md"} and then prose.', + true + ) + expect(hasPendingTag).toBe(false) + }) + + it('keeps a valid tag whose close an earlier broken tag would borrow', () => { + // The first opener misspells its close, so it reaches forward and matches + // the SECOND tag's close, swallowing a perfectly good resource into one + // literal span. Resuming past the opener re-scans the interior instead. + const raw = + 'See {"type":"file","path":"a.md"}\n' + + 'and a real one: {"type":"file","path":"b.md","title":"b"}' + const { segments } = parseSpecialTags(raw, false) + expect(segments.some((s) => s.type === 'workspace_resource')).toBe(true) + expect(segments.map((s) => ('content' in s ? s.content : '')).join('')).toContain( + '' + ) + }) + it('still renders a matched pair whose body IS valid', () => { const raw = 'see {"type":"file","path":"files/a.md","title":"a.md"} ok' @@ -275,10 +330,10 @@ describe('parseSpecialTags with ', () => { it('bails when a foreign closing tag appears inside the body', () => { // Tags never nest, so a close for a different tag proves the opener was text. - const { hasPendingTag } = parseSpecialTags( - 'see [{"title":"a","description":"b"}] more', - true - ) + // The array is left UNCLOSED on purpose: with a closed value the JSON rule + // would independently reject the trailing content, and this test would still + // pass with the nesting rule deleted. Unclosed, only nesting can explain it. + const { hasPendingTag } = parseSpecialTags('see [{"title":"a" more', true) expect(hasPendingTag).toBe(false) }) @@ -349,18 +404,6 @@ describe('parseSpecialTags with ', () => { expect(hasPendingTag).toBe(true) expect(segments).toEqual([{ type: 'text', content: 'Let me ask. ' }]) }) - - it('drops a question tag with an invalid body but keeps surrounding text', () => { - const { segments, hasPendingTag } = parseSpecialTags( - 'Before. {"type":"single_select"} After.', - false - ) - expect(hasPendingTag).toBe(false) - expect(segments).toEqual([ - { type: 'text', content: 'Before. ' }, - { type: 'text', content: ' After.' }, - ]) - }) }) describe('service_account credential tag', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 1c39daaa263..7dbae91aae6 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -464,15 +464,6 @@ function parseSpecialTagData( return null } -/** - * Parses inline special tags (``, ``, ``) from streamed - * text content. Complete tags are extracted into typed segments; incomplete - * tags (still streaming) are suppressed from display and flagged via - * `hasPendingTag` so the caller can show a loading indicator. - * - * Trailing partial opening tags (e.g. `` is exactly the case that matters. @@ -480,18 +471,36 @@ function parseSpecialTagData( const TAG_SHAPED_MARKER = /<\/?[a-zA-Z][\w-]*>/ /** - * Tags whose body must be JSON. `thinking` is the exception — its body is prose - * (see {@link parseTextTagBody}), so a non-JSON body there says nothing about - * whether a close is still coming. + * The one tag whose body is prose rather than JSON (see {@link parseTextTagBody}), + * so a non-JSON body there says nothing about whether a close is still coming. */ -const JSON_BODY_TAG_NAMES = new Set<(typeof SPECIAL_TAG_NAMES)[number]>([ - 'options', - 'usage_upgrade', - 'credential', - 'mothership-error', - 'workspace_resource', - 'question', -]) +const PROSE_BODY_TAG_NAME: (typeof SPECIAL_TAG_NAMES)[number] = 'thinking' + +/** + * Tags whose body must be JSON. + * + * Derived from {@link SPECIAL_TAG_NAMES} rather than hand-listed: a new tag is + * JSON-bodied by default, so forgetting to update this set cannot silently + * downgrade it to the weaker prose heuristics. Opting a tag out is an explicit + * edit to {@link PROSE_BODY_TAG_NAME}. + */ +const JSON_BODY_TAG_NAMES: ReadonlySet<(typeof SPECIAL_TAG_NAMES)[number]> = new Set( + SPECIAL_TAG_NAMES.filter((name) => name !== PROSE_BODY_TAG_NAME) +) + +/** + * How much of an unclosed body to inspect per parse. + * + * Both rules in {@link unclosedTagCannotResolve} decide on their FIRST piece of + * evidence — the first foreign marker, or the first character that breaks JSON + * viability — so a bounded window reaches the same verdict as the full remainder + * for any real payload. Unbounded, the check is O(remaining length) and runs + * once per opener inside a parse that re-runs for every streamed chunk, so a + * long reply repeatedly mentioning a tag name in prose costs seconds of + * main-thread time. Evidence past the window only defers the decision to a later + * chunk, which is the same conservative direction the rules already take. + */ +const MAX_UNCLOSED_BODY_SCAN = 4096 /** * Strip the contents of JSON string literals from `body`, replacing them with @@ -544,7 +553,15 @@ function blankJsonStringLiterals(body: string): string { * do not affect the depth count. */ function isViableJsonPrefix(body: string): boolean { - const scannable = blankJsonStringLiterals(body) + return isViableJsonPrefixOf(blankJsonStringLiterals(body)) +} + +/** + * {@link isViableJsonPrefix} for a body whose string literals are already + * blanked. Callers that had to blank the body for their own scan pass the result + * straight through instead of paying for a second pass over the same text. + */ +function isViableJsonPrefixOf(scannable: string): boolean { if (scannable.trim() === '') return true const firstChar = scannable.trimStart().charAt(0) @@ -577,8 +594,10 @@ function isViableJsonPrefix(body: string): boolean { * * 1. Tags never nest. Any other special-tag marker inside the body — opening or * closing — means this opener was literal text, not a real tag. - * 2. JSON-bodied tags must start with `{` or `[`. The first non-space character - * settles it, so prose after the marker is caught on the very next chunk. + * 2. A JSON body must stay a viable JSON prefix. Depth is tracked rather than + * testing the first character alone, so a body whose top-level value has + * already closed is caught the moment stray content follows it — including a + * misspelled close like ``, which no marker rule sees. * * Both are conservative: they only fire on content that could not have parsed. * A false positive would merely show text early that a later chunk resolves @@ -588,22 +607,44 @@ function unclosedTagCannotResolve( tagName: (typeof SPECIAL_TAG_NAMES)[number], body: string ): boolean { + const pending = dropArrivingClose(body, ``) // For a JSON-bodied tag, ignore markers inside string literals: quoting tag // syntax is legitimate content, and treating it as evidence would bail on a // tag that goes on to close correctly — showing raw JSON that then snaps into // a rendered card. - const scannable = JSON_BODY_TAG_NAMES.has(tagName) ? blankJsonStringLiterals(body) : body + const isJsonBodied = JSON_BODY_TAG_NAMES.has(tagName) + const scannable = isJsonBodied ? blankJsonStringLiterals(pending) : pending for (const name of SPECIAL_TAG_NAMES) { // A close for this tag is absent by definition here, so this catches a // FOREIGN close; the open check catches nesting, including self-nesting. if (scannable.includes(``) || scannable.includes(`<${name}>`)) return true } - if (JSON_BODY_TAG_NAMES.has(tagName) && !isViableJsonPrefix(body)) return true + if (isJsonBodied && !isViableJsonPrefixOf(scannable)) return true return false } +/** + * Drop a trailing fragment that could still grow into `closeTag`. + * + * Mid-stream the closing marker arrives a character at a time, so a body sits at + * `]` completes. That fragment is an + * arriving close, not stray content — counting it as fatal is what made a + * perfectly valid tag show its raw payload as text until the final `>` landed. + * + * Only a fragment at the very END is dropped, so evidence that the close is + * genuinely wrong still lands immediately: a misspelled `` + * is not a prefix of ``, and a truncated ` 0; n--) { + if (body.endsWith(closeTag.slice(0, n))) return body.slice(0, -n) + } + return body +} + /** * How one opening tag resolved. Naming the four outcomes is the point: the * parser previously decided each case inline, which is how "drop it" quietly @@ -620,30 +661,76 @@ type TagResolution = | { outcome: 'pending' } /** - * True when a failed body was never an attempted payload — so the markers were - * literal text and the span must be shown rather than swallowed. + * Why a failed body was never an attempted payload — so the markers were literal + * text and the span must be shown rather than swallowed. `null` means the body + * really was a payload that failed its shape guard. + * + * The two reasons resume differently, which is why they are distinguished + * rather than collapsed into a boolean (see {@link resolveTagAt}). + */ +type LiteralTextReason = + /** The body carries tag markers, so the close we matched belongs elsewhere. */ + | 'foreign-markers' + /** The tag wrapped prose that was never JSON to begin with. */ + | 'never-a-payload' + +function literalTextReason( + tagName: (typeof SPECIAL_TAG_NAMES)[number], + body: string +): LiteralTextReason | null { + const isJsonBodied = JSON_BODY_TAG_NAMES.has(tagName) + // Markers inside a JSON string are content, not evidence — a `` may + // legitimately quote tag syntax in its prompt. Scanning the raw body here + // would classify a broken payload as literal text and render it as raw JSON, + // which is exactly what `discard` exists to prevent. Mirrors the same blanking + // in unclosedTagCannotResolve, which judges the same body mid-stream. + const scannable = isJsonBodied ? blankJsonStringLiterals(body) : body + if (TAG_SHAPED_MARKER.test(scannable)) return 'foreign-markers' + if (isJsonBodied && !isViableJsonPrefixOf(scannable)) return 'never-a-payload' + return null +} + +/** + * `content.indexOf(needle, from)` memoized per needle. * - * Either the close we matched belongs to a different opener (the body carries - * tag markers), or the tag wrapped prose that was never JSON to begin with. + * The opener scan and the close lookup search the same handful of markers over + * and over as the cursor advances. A needle absent from the message resolves to + * -1 once and is never searched again; a present one is re-searched only when + * the cursor passes its last hit. Unmemoized, each lookup rescans to the end of + * the buffer for every opener, which is quadratic on a message that mentions a + * tag name many times — and this parse re-runs for every streamed chunk. + * + * Safe because `from` only ever increases within a parse: a cached -1 stays -1, + * and a cached hit at or after `from` is still the first hit at or after `from`. */ -function bodyIsLiteralText(tagName: (typeof SPECIAL_TAG_NAMES)[number], body: string): boolean { - if (TAG_SHAPED_MARKER.test(body)) return true - return JSON_BODY_TAG_NAMES.has(tagName) && !isViableJsonPrefix(body) +function memoizedIndexOf( + cache: Map, + content: string, + needle: string, + from: number +): number { + const cached = cache.get(needle) + if (cached !== undefined && (cached === -1 || cached >= from)) return cached + const idx = content.indexOf(needle, from) + cache.set(needle, idx) + return idx } function resolveTagAt( content: string, openIndex: number, tagName: (typeof SPECIAL_TAG_NAMES)[number], - isStreaming: boolean + isStreaming: boolean, + closeCache: Map ): TagResolution { const openTag = `<${tagName}>` const closeTag = `` const bodyStart = openIndex + openTag.length - const closeIdx = content.indexOf(closeTag, bodyStart) + const closeIdx = memoizedIndexOf(closeCache, content, closeTag, bodyStart) if (closeIdx === -1) { - if (isStreaming && !unclosedTagCannotResolve(tagName, content.slice(bodyStart))) { + const inspectable = content.slice(bodyStart, bodyStart + MAX_UNCLOSED_BODY_SCAN) + if (isStreaming && !unclosedTagCannotResolve(tagName, inspectable)) { return { outcome: 'pending' } } // Nothing can close it, so only the opener itself is literal. Resuming just @@ -658,7 +745,16 @@ function resolveTagAt( const parsed = parseSpecialTagData(tagName, body) if (parsed) return { outcome: 'segment', segment: parsed, resumeAt } - if (bodyIsLiteralText(tagName, body)) return { outcome: 'literal', resumeAt } + const reason = literalTextReason(tagName, body) + if (reason === 'foreign-markers') { + // A marker in the body proves the close we matched opened somewhere else — + // this opener reached past its own missing close and borrowed a later tag's. + // Resuming past the OPENER instead of past that borrowed close re-scans the + // interior, so the genuine tag inside still renders instead of being + // swallowed into one literal span. + return { outcome: 'literal', resumeAt: bodyStart } + } + if (reason === 'never-a-payload') return { outcome: 'literal', resumeAt } // A well-formed value that failed its shape guard is a broken emission from // the agent; showing the user raw JSON there would be worse than nothing. @@ -667,7 +763,10 @@ function resolveTagAt( /** * Splits streamed text into renderable segments, extracting complete special - * tags and deciding what to do with the ones that never resolve. + * tags and deciding what to do with the ones that never resolve. Incomplete + * tags are suppressed and flagged via `hasPendingTag` so the caller can show a + * loading indicator, and a trailing partial opening marker (`() + const closeCache = new Map() + while (cursor < content.length) { let nearestStart = -1 let nearestTagName: (typeof SPECIAL_TAG_NAMES)[number] | '' = '' for (const name of SPECIAL_TAG_NAMES) { - const idx = content.indexOf(`<${name}>`, cursor) + const idx = memoizedIndexOf(openerCache, content, `<${name}>`, cursor) if (idx !== -1 && (nearestStart === -1 || idx < nearestStart)) { nearestStart = idx nearestTagName = name @@ -717,7 +819,7 @@ export function parseSpecialTags(content: string, isStreaming: boolean): ParsedS pushText(content.slice(cursor, nearestStart)) - const resolution = resolveTagAt(content, nearestStart, nearestTagName, isStreaming) + const resolution = resolveTagAt(content, nearestStart, nearestTagName, isStreaming, closeCache) if (resolution.outcome === 'pending') { hasPendingTag = true