From 7751c6e4f817e3d61b04776f8d5d38abee0186ed Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Sat, 26 Sep 2026 10:01:15 -0400 Subject: [PATCH 1/3] fix: keep input marking through filters, tojson and join; undefined member access (#30, #31, #33) --- CHANGELOG.md | 10 + README.md | 2 + lib/src/ast/nodes.dart | 131 +++--- lib/src/runtime/builtins.dart | 549 ++++++++++++++++++-------- lib/src/types/marking.dart | 51 +++ test/src/ast/nodes_test.dart | 151 +++++++ test/src/input_escaping_test.dart | 215 ++++++++++ test/src/runtime/builtins_test.dart | 161 +++++++- test/src/types/jinja_string_test.dart | 48 ++- 9 files changed, 1075 insertions(+), 243 deletions(-) create mode 100644 lib/src/types/marking.dart create mode 100644 test/src/input_escaping_test.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a4c918..7d5a4e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## Unreleased + +- Fixed a `{% filter %}` block, and a filter on the output of a block `set`, a macro or `caller()`, getting `JinjaString.user` text already escaped: with `x = JinjaString.user('')`, `{% filter upper %}{{ x }}{% endfilter %}` is now `<B>`, not `<B>`, and `{% filter length %}{{ x }}{% endfilter %}` is `3`, not `9`, as in llama.cpp. Input text is now escaped once, when the template outputs it. So `| safe` on such output, and `{% filter safe %}`, now turn escaping off for the input in it, as `{{ x | safe }}` does. A block whose output also contains `| safe` input text is still escaped before the filter runs, as `~` does. +- Fixed `tojson` and `join` dropping the input marking, so `JinjaString.user` text in their output was not escaped: `{{ {"k": x} | tojson }}` is now `{"k": "<b>"}` and `{{ [x, ""] | join }}` is `<b>`. Only input text is escaped, after JSON escaping, in values, dict keys and separators; the JSON's own quotes and template text are not. `tojson` output with input text in it is no longer safe (`is escaped`). +- Fixed input text being output unescaped by string indexing, `first`, `last`, `reverse`, `list` and `for` on a string, `split` and `rsplit` pieces, `strip`, `lstrip` and `rstrip` with characters, `truncate`, `str()`, `~`, `+` and `format` with a list or dict, `string`, `upper`, `lower`, `title`, `capitalize`, `trim`, `replace` and `indent` on a list or dict, `replace`, `indent` and `format` on text that is only partly input, input passed as a `replace` replacement, input dict keys in a `for` loop, and an input `strftime_now` format. +- Changed `replace` and `indent` on input text to leave the template text they add unescaped: `x | replace('b', '')` is now `<>`, not `<<i>>`, and `x | indent('> ', true)` no longer escapes `> `. Each character keeps the marking of the text it came from; llama.cpp marks all of the result as input when all of the source is. +- Fixed `| safe` text being escaped when passed to `format`, or to `replace` or `indent` on input text, sliced with a step or split by `unique`, and escaped twice when it held escaped input: `'{}'.format(x ~ (y | safe))` is now `<b>`, not `&lt;b&gt;`. +- Fixed member access on an undefined value giving a function, which printed as `` and was `defined`: for an undefined `y`, `{{ y.text }}` and `{{ y[0]['text'] }}` now print nothing and are not `defined`, as in llama.cpp. As in llama.cpp, `y.upper`, `y.length` and the other filters that llama.cpp gives undefined are functions returning an empty value (`y.length()` is `0`, `y.default(1)` is `1`) that print nothing, and calling any other member, such as `y.get()`, throws. Devstral Small 2507 no longer prints `` for an assistant turn without `content`. +- Changed a number subscript of none, as in `x[0]` or `x[0]['text']` for `x = none`, to throw `Cannot access property with non-string: got Integer`, as llama.cpp does; it gave undefined. Devstral Small 2507 with an assistant turn whose content is none now throws, as in llama.cpp. + ## 1.2.0 - Rewrote the README and example around rendering a chat template, and updated the pubspec description and topics. diff --git a/README.md b/README.md index 920dd82..add8893 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,8 @@ Real templates come from the `chat_template` field of a model's `tokenizer_confi Wrap untrusted values in `JinjaString.user`. They are escaped when rendered; a plain `String` is treated as template text and is not. +Input text keeps its marking through filters, `~`, loops, macros and `{% set %}` and `{% filter %}` blocks, and is escaped once, when the template outputs it. So a filter sees it as it was passed in: `{% filter length %}{{ name }}{% endfilter %}` counts the characters of `name`, not of its escaped form. `tojson` and `join` escape only the input text in their output, not the JSON's quotes or the template's separator. `| safe` and `markSafe()` turn escaping off. + ```dart import 'package:dinja/dinja.dart'; diff --git a/lib/src/ast/nodes.dart b/lib/src/ast/nodes.dart index 57deda7..edf328f 100644 --- a/lib/src/ast/nodes.dart +++ b/lib/src/ast/nodes.dart @@ -1,5 +1,6 @@ import '../types/value.dart'; import '../types/jinja_string.dart'; +import '../types/marking.dart'; import '../types/repr.dart'; import 'dart:math' as math; import '../runtime/context.dart'; @@ -9,7 +10,7 @@ import '../lexer.dart'; // for Token /// Thrown by `break` and `continue`. Carries the output that the loop pass /// rendered before the statement, which the loop keeps. abstract class LoopSignal { - final List parts = []; + final List output = []; } class BreakSignal extends LoopSignal {} @@ -48,53 +49,53 @@ class Program extends Statement { @override String get type => 'Program'; + /// Renders the template. Input-marked text is escaped here, once. @override JinjaValue execute(Context ctx) { - return execStatements(body, ctx); + final result = execStatements(body, ctx) as JinjaStringValue; + return JinjaStringValue(result.value.escape()); } } -// ... helper to execute list of statements and gather output ... -/// Executes a list of [stmts] within the given [ctx] and aggregates their results. +/// Executes a list of [stmts] within the given [ctx] and joins their output. /// /// If a statement returns a string value, it is added to the output. /// Non-string values (except None and Undefined) are converted to strings. +/// Input-marked text stays unescaped, so a filter applied to the output sees +/// it as it was passed in; see [_joinOutput]. JinjaValue execStatements(List stmts, Context ctx) { - List parts = []; + final output = []; try { for (final stmt in stmts) { final val = stmt.execute(ctx); if (val is JinjaStringValue) { - if (val.isSafe) { - parts.addAll(val.value.parts); - } else { - // Check parts for input/unsafe content - for (final part in val.value.parts) { - if (part.isInput) { - final escaped = part.val - .replaceAll('&', '&') - .replaceAll('<', '<') - .replaceAll('>', '>') - .replaceAll('"', '"') - .replaceAll("'", '''); - // print('DEBUG escaping: ${part.val} -> $escaped'); - parts.add( - JinjaStringPart(escaped, part.isInput), - ); // Escaped -> safe/template - } else { - parts.add(part); - } - } - } + output.add(val.value); } else if (!val.isNone && !val.isUndefined) { - parts.addAll(reprOf(val).escape().parts); + output.add(reprOf(val)); } } } on LoopSignal catch (signal) { - signal.parts.insertAll(0, parts); + signal.output.insertAll(0, output); rethrow; } - return JinjaStringValue(JinjaString(parts, isSafe: true)); + return JinjaStringValue(_joinOutput(output)); +} + +/// Joins rendered [pieces]. +/// +/// Input-marked text stays unescaped unless a piece is a safe string with +/// input-marked text, whose text is final: then every piece is escaped and +/// the result is safe. +JinjaString _joinOutput(List pieces) { + final safeInput = pieces.any( + (p) => p.isSafe && p.parts.any((part) => part.isInput), + ); + if (pieces.any(hasRawInput) && !safeInput) { + return JinjaString([for (final p in pieces) ...p.parts]); + } + return JinjaString([ + for (final p in pieces) ...p.escape().parts, + ], isSafe: true); } /// Executes [stmts] whose output is captured rather than printed, as in a @@ -104,7 +105,7 @@ JinjaValue _execCaptured(List stmts, Context ctx) { try { return execStatements(stmts, ctx); } on LoopSignal catch (signal) { - signal.parts.clear(); + signal.output.clear(); rethrow; } } @@ -194,22 +195,22 @@ class ForStatement extends Statement { // Sort keys for deterministic output if needed, though Jinja2 doesn't strictly guarantee order, usually tests expect sorted keys or we should sort. // Standard Jinja2 iterates keys. // llama.cpp tests seem to expect keys. - final keys = iterableVal.asMap.keys.toList(); - // keys.sort(); // Optional: sort keys? Dinja existing tests expect 'a,b'. Maps in Dart preserve insertion order? LinkedHashMap does. - // If the map came from JSON/Literal, it's LinkedHashMap. - - items = keys.map((k) => JinjaStringValue.fromString(k)).toList(); + // Keys with the same string form are iterated once, as in asMap. + final keys = { + for (final k in iterableVal.items.keys) k.toString(): k, + }; + items = [ + for (final MapEntry(:key, :value) in keys.entries) + value is JinjaStringValue && hasRawInput(value.value) + ? value + : JinjaStringValue.fromString(key), + ]; } else if (iterableVal is JinjaList) { items = iterableVal.items; } else if (iterableVal is JinjaTuple) { items = iterableVal.items; } else if (iterableVal is JinjaStringValue) { - // Iterate chars - items = iterableVal.value.parts - .map((p) => p.val) - .expand((s) => s.split('')) - .map((c) => JinjaStringValue.fromString(c)) - .toList(); + items = charsOf(iterableVal.value); } // Filter items first if testExpr @@ -228,7 +229,7 @@ class ForStatement extends Statement { } // Loop - List parts = []; + final output = []; // As in Jinja2 and llama.cpp, the else block runs unless a pass finishes // without `break` or `continue`. var iterated = false; @@ -257,12 +258,12 @@ class ForStatement extends Statement { try { final result = execStatements(body, loopCtx); - if (result is JinjaStringValue) parts.addAll(result.value.parts); + if (result is JinjaStringValue) output.add(result.value); iterated = true; } on ContinueSignal catch (signal) { - parts.addAll(signal.parts); + output.addAll(signal.output); } on BreakSignal catch (signal) { - parts.addAll(signal.parts); + output.addAll(signal.output); break; } } @@ -270,15 +271,15 @@ class ForStatement extends Statement { if (!iterated) { try { final result = execStatements(defaultBlock, ctx); - if (result is JinjaStringValue) parts.addAll(result.value.parts); + if (result is JinjaStringValue) output.add(result.value); } on LoopSignal catch (signal) { // A `break` or `continue` in the else block belongs to an enclosing // loop and keeps the output of this one, as in Jinja2. - signal.parts.insertAll(0, parts); + signal.output.insertAll(0, output); rethrow; } } - return JinjaStringValue(JinjaString(parts, isSafe: true)); + return JinjaStringValue(_joinOutput(output)); } void _bindLoopVar(Context ctx, Expression loopVar, JinjaValue item) { @@ -789,14 +790,15 @@ class MemberExpression extends Expression { return JinjaStringValue(s.substring(start, stop)); } + final raw = rawOf(s); final parts = []; if (step > 0) { for (int i = start; i < stop; i += step) { - if (i >= 0 && i < len) parts.addAll(s[i].parts); + if (i >= 0 && i < len) parts.addAll(raw[i].parts); } } else if (step < 0) { for (int i = start; i > stop; i += step) { - if (i >= 0 && i < len) parts.addAll(s[i].parts); + if (i >= 0 && i < len) parts.addAll(raw[i].parts); } } return JinjaStringValue(JinjaString(parts)); @@ -812,6 +814,14 @@ class MemberExpression extends Expression { if (!computed && prop.asInt < 0) { throw Exception('Static member property cannot be negative'); } + // As in llama.cpp, a subscript of undefined is undefined, and none + // takes only a string subscript. + if (obj.isUndefined || prop.isUndefined) return const JinjaUndefined(); + if (obj.isNone && prop is! JinjaStringValue) { + throw Exception( + 'Cannot access property with non-string: got ${prop.typeName}', + ); + } if (obj is JinjaMap) { // Bracket access: check keys FIRST @@ -851,7 +861,7 @@ class MemberExpression extends Expression { String s = obj.value.toString(); if (idx < 0) idx += s.length; if (idx >= 0 && idx < s.length) { - return JinjaStringValue.fromString(s[idx]); + return JinjaStringValue(rawOf(obj.value)[idx]); } } } @@ -1241,13 +1251,18 @@ class TestExpression extends Expression { } JinjaStringValue _concat(JinjaValue l, JinjaValue r) { - var left = l is JinjaStringValue ? l.value : JinjaString.template('$l'); - var right = r is JinjaStringValue ? r.value : JinjaString.template('$r'); - if (left.isSafe != right.isSafe) { - left = left.escape(); - right = right.escape(); - } - return JinjaStringValue(left + right); + final left = _operand(l); + final right = _operand(r); + if (left.isSafe == right.isSafe) return JinjaStringValue(left + right); + final joined = _joinOutput([left, right]); + if (!joined.isSafe) return JinjaStringValue(joined); + return JinjaStringValue(left.escape() + right.escape()); +} + +JinjaString _operand(JinjaValue v) { + if (v is JinjaStringValue) return v.value; + final s = stringOf(v); + return hasRawInput(s) ? s : JinjaString.template('$v'); } /// Represents a selection expression used in filtering iterations. diff --git a/lib/src/runtime/builtins.dart b/lib/src/runtime/builtins.dart index aa9816d..56d3e91 100644 --- a/lib/src/runtime/builtins.dart +++ b/lib/src/runtime/builtins.dart @@ -1,6 +1,7 @@ // ignore_for_file: non_constant_identifier_names import '../types/value.dart'; import '../types/jinja_string.dart'; +import '../types/marking.dart'; import '../types/repr.dart'; import 'strftime.dart'; import 'dart:convert'; @@ -64,26 +65,12 @@ final Map globalFilters = { 'format': _formatFilter, 'string': _string, 'title': (args, kwargs) => - (_resolveStringMember( - args.isNotEmpty && args[0] is JinjaStringValue - ? args[0] as JinjaStringValue - : JinjaStringValue.fromString( - args.isNotEmpty ? args[0].toString() : '', - ), - 'title', - ) - as JinjaFunction) - .handler(args, kwargs), + (_resolveStringMember(_asString(args), 'title') as JinjaFunction).handler( + args, + kwargs, + ), 'capitalize': (args, kwargs) => - (_resolveStringMember( - args.isNotEmpty && args[0] is JinjaStringValue - ? args[0] as JinjaStringValue - : JinjaStringValue.fromString( - args.isNotEmpty ? args[0].toString() : '', - ), - 'capitalize', - ) - as JinjaFunction) + (_resolveStringMember(_asString(args), 'capitalize') as JinjaFunction) .handler(args, kwargs), 'truncate': _truncate, 'wordcount': _wordcount, @@ -217,6 +204,18 @@ JinjaValue? resolveMember(JinjaValue obj, String name) { // Implementations +/// The first argument as a string value, keeping the input marking of the +/// strings in a list or dict. +JinjaStringValue _asString(List args) { + if (args.isEmpty) return JinjaStringValue.fromString(''); + final v = args[0]; + if (v is JinjaStringValue) return v; + final s = stringOf(v); + return hasRawInput(s) + ? JinjaStringValue(s) + : JinjaStringValue.fromString(v.toString()); +} + JinjaValue _replaceFilter( List args, Map kwargs, @@ -224,9 +223,62 @@ JinjaValue _replaceFilter( if (args.isEmpty) return const JinjaStringValue(JinjaString([])); final obj = args[0]; if (args.length < 3) return obj; - final oldVal = args[1].toString(); - final newVal = args[2].toString(); - return _derived(obj, obj.toString().replaceAll(oldVal, newVal)); + final newVal = stringOf(args[2]); + return _derivedFrom( + obj, + _replaceIn(stringOf(obj), args[1].toString(), newVal, -1), + [newVal], + ); +} + +/// [s] with [old] replaced by [rep]: everywhere, as `String.replaceAll`, or +/// at most [count] times when it is not negative. Each character keeps the +/// input marking of the string it came from. +JinjaString _replaceIn(JinjaString s, String old, JinjaString rep, int count) { + s = rawOf(s); + rep = rawOf(rep); + if (count < 0) { + final text = s.toString(); + var prev = 0; + final parts = []; + for (final m in old.allMatches(text)) { + parts + ..addAll(s.substring(prev, m.start).parts) + ..addAll(rep.parts); + prev = m.end; + } + return JinjaString([...parts, ...s.substring(prev).parts]); + } + var res = s; + var start = 0; + for (var total = 0; total < count; total++) { + final idx = res.toString().indexOf(old, start); + if (idx == -1) break; + res = JinjaString([ + ...res.substring(0, idx).parts, + ...rep.parts, + ...res.substring(idx + old.length).parts, + ]); + start = idx + rep.length; + } + return res; +} + +/// [result], computed from [source] and the [inserted] strings. +/// +/// When any of them has input-marked text still to escape, each character +/// keeps the input marking of where it came from, so input text is escaped +/// and template text is not. Otherwise [result] is marked as `_derived` +/// marks it. +JinjaStringValue _derivedFrom( + JinjaValue source, + JinjaString result, [ + List inserted = const [], +]) { + if (hasRawInput(stringOf(source)) || inserted.any(hasRawInput)) { + return JinjaStringValue(joinRaw([result])); + } + return _derived(source, result.toString()); } JinjaStringValue _derived(JinjaValue source, String text) { @@ -239,8 +291,19 @@ JinjaStringValue _derived(JinjaValue source, String text) { ); } -JinjaStringValue _retext(JinjaStringValue source, String text) { - if (text.length != source.value.length) return _derived(source, text); +/// [text], a case change of [source], with the input marking of [source]. +/// +/// When the change alters the length and [source] has input-marked text to +/// escape, [perPart] computes it part by part instead. +JinjaStringValue _retext( + JinjaStringValue source, + String text, + JinjaString Function() perPart, +) { + if (text.length != source.value.length) { + if (hasRawInput(source.value)) return JinjaStringValue(perPart()); + return _derived(source, text); + } final parts = []; var start = 0; for (final part in source.value.parts) { @@ -293,31 +356,59 @@ JinjaValue _tojson(List args, Map kwargs) { final separators = separatorsArg is JinjaList || separatorsArg is JinjaTuple ? separatorsArg!.asList : const []; - final out = StringBuffer(); + final out = _JsonOut(); _writeJson( out, args[0], 0, indent: indent, itemSep: separators.isNotEmpty - ? separators[0].toString() - : (indent < 0 ? ', ' : ','), - keySep: separators.length > 1 ? separators[1].toString() : ': ', + ? stringOf(separators[0]) + : JinjaString.template(indent < 0 ? ', ' : ','), + keySep: separators.length > 1 + ? stringOf(separators[1]) + : JinjaString.template(': '), ensureAscii: arg('ensure_ascii', 1)?.asBool ?? false, sortKeys: arg('sort_keys', 4)?.asBool ?? false, ); - return JinjaStringValue( - JinjaString([JinjaStringPart(out.toString(), false)], isSafe: true), - ); + final json = joinRaw(out.pieces); + // Without input-marked text the output is safe template text, as before. + if (!json.parts.any((p) => p.isInput)) { + return JinjaStringValue( + JinjaString([JinjaStringPart(json.toString(), false)], isSafe: true), + ); + } + return JinjaStringValue(json); +} + +/// JSON output that keeps the input marking of the strings written to it. +class _JsonOut { + final List pieces = []; + + void write(Object text) => pieces.add(JinjaString.template('$text')); + + void writeString(JinjaString s, bool ensureAscii) { + write('"'); + pieces.add( + JinjaString([ + for (final p in rawOf(s).parts) + JinjaStringPart( + jsonEscape(p.val, ensureAscii: ensureAscii), + p.isInput, + ), + ]), + ); + write('"'); + } } void _writeJson( - StringBuffer out, + _JsonOut out, JinjaValue v, int level, { required int indent, - required String itemSep, - required String keySep, + required JinjaString itemSep, + required JinjaString keySep, required bool ensureAscii, required bool sortKeys, }) { @@ -335,7 +426,7 @@ void _writeJson( for (var i = 0; i < items.length; i++) { out.write(pad * (level + 1)); writeItem(items[i]); - if (i < items.length - 1) out.write(itemSep); + if (i < items.length - 1) out.pieces.add(itemSep); out.write(newline); } out.write(pad * level); @@ -361,16 +452,19 @@ void _writeJson( } else if (v is JinjaFloat) { out.write(_formatDouble(v.value)); } else if (v is JinjaStringValue) { - out.write('"${jsonEscape(v.toString(), ensureAscii: ensureAscii)}"'); + out.writeString(v.value, ensureAscii); } else if (v is JinjaList || v is JinjaTuple) { writeItems('[', ']', v.asList, writeValue); } else if (v is JinjaMap) { final entries = v.items.entries - .map((e) => MapEntry(e.key.toString(), e.value)) + .map((e) => MapEntry(stringOf(e.key), e.value)) .toList(); - if (sortKeys) entries.sort((a, b) => a.key.compareTo(b.key)); - writeItems('{', '}', entries, (MapEntry e) { - out.write('"${jsonEscape(e.key, ensureAscii: ensureAscii)}"$keySep'); + if (sortKeys) { + entries.sort((a, b) => a.key.toString().compareTo(b.key.toString())); + } + writeItems('{', '}', entries, (MapEntry e) { + out.writeString(e.key, ensureAscii); + out.pieces.add(keySep); writeValue(e.value); }); } else { @@ -462,15 +556,7 @@ JinjaValue _list(List args, Map kwargs) { final arg = args[0]; if (arg is JinjaList) return arg; if (arg is JinjaTuple) return JinjaList(List.from(arg.items)); - if (arg is JinjaStringValue) { - return JinjaList( - arg.value.parts - .map((p) => p.val) - .expand((s) => s.split('')) - .map((c) => JinjaStringValue.fromString(c)) - .toList(), - ); - } + if (arg is JinjaStringValue) return JinjaList(charsOf(arg.value)); if (arg is JinjaMap) { // list(dict) -> keys return JinjaList( @@ -519,7 +605,7 @@ JinjaValue _float(List args, Map kwargs) { JinjaValue _str(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); - return JinjaStringValue.fromString(args[0].toString()); + return JinjaStringValue(joinRaw([stringOf(args[0])])); } JinjaValue _len(List args, Map kwargs) { @@ -537,7 +623,7 @@ JinjaValue _first(List args, Map kwargs) { final v = args[0]; if (v is JinjaList && v.items.isNotEmpty) return v.items.first; if (v is JinjaStringValue && v.value.length > 0) { - return JinjaStringValue.fromString(v.value.toString()[0]); + return JinjaStringValue(rawOf(v.value)[0]); } if (v is JinjaTuple && v.items.isNotEmpty) return v.items.first; return const JinjaUndefined(); @@ -548,9 +634,7 @@ JinjaValue _last(List args, Map kwargs) { final v = args[0]; if (v is JinjaList && v.items.isNotEmpty) return v.items.last; if (v is JinjaStringValue && v.value.length > 0) { - return JinjaStringValue.fromString( - v.value.toString().substring(v.value.length - 1), - ); + return JinjaStringValue(rawOf(v.value)[v.value.length - 1]); } if (v is JinjaTuple && v.items.isNotEmpty) return v.items.last; return const JinjaUndefined(); @@ -739,11 +823,7 @@ JinjaValue _unique(List args, Map kwargs) { if (collection is JinjaList || collection is JinjaTuple) { items = collection.asList; } else if (collection is JinjaStringValue) { - items = [ - for (final part in collection.value.parts) - for (final char in part.val.split('')) - JinjaStringValue(JinjaString([JinjaStringPart(char, part.isInput)])), - ]; + items = charsOf(collection.value); } else if (collection is JinjaMap) { items = collection.items.keys.toList(); } else if (collection.isNone) { @@ -783,8 +863,8 @@ JinjaValue _reverse(List args, Map kwargs) { final v = args[0]; if (v is JinjaList) return JinjaList(v.items.reversed.toList()); if (v is JinjaStringValue) { - return JinjaStringValue.fromString( - v.value.toString().split('').reversed.join(''), + return JinjaStringValue( + joinRaw(charsOf(v.value).reversed.map((c) => stringOf(c))), ); } return v; @@ -976,11 +1056,13 @@ JinjaValue _join(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); final collection = args[0]; if (collection is! JinjaList && collection is! JinjaTuple) { - return JinjaStringValue.fromString(collection.toString()); + return JinjaStringValue(joinRaw([stringOf(collection)])); } - final delimiter = - (kwargs['d'] ?? (args.length > 1 ? args[1] : null))?.toString() ?? ''; + final delimiterArg = kwargs['d'] ?? (args.length > 1 ? args[1] : null); + final delimiter = delimiterArg == null + ? JinjaString.template('') + : stringOf(delimiterArg); final attribute = kwargs['attribute']?.toString() ?? (args.length > 2 ? args[2].toString() : null); @@ -989,14 +1071,16 @@ JinjaValue _join(List args, Map kwargs) { ? collection.items : (collection as JinjaTuple).items; - final strings = items - .map((e) { - final val = attribute != null ? _resolveAttribute(e, attribute) : e; - return val.toString(); - }) - .join(delimiter); - - return JinjaStringValue.fromString(strings); + return JinjaStringValue( + joinRaw([ + for (var i = 0; i < items.length; i++) ...[ + if (i > 0) delimiter, + stringOf( + attribute != null ? _resolveAttribute(items[i], attribute) : items[i], + ), + ], + ]), + ); } JinjaValue _safe(List args, Map kwargs) { @@ -1455,13 +1539,14 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { final maxsplit = kwargs['maxsplit']?.asInt ?? (args.length > 1 ? args[1].asInt : -1); - String s = obj.value.toString(); - List parts; + final src = rawOf(obj.value); + final s = src.toString(); + List parts; if (delimiter == null || delimiter == ' ') { // split by whitespace - parts = s.trim().split(RegExp(r'\s+')); + parts = _splitWhitespace(src); if (maxsplit >= 0 && parts.length > maxsplit + 1) { - final rest = parts.sublist(maxsplit).join(' '); + final rest = _joinSpaces(parts.sublist(maxsplit)); parts = parts.sublist(0, maxsplit)..add(rest); } } else { @@ -1472,20 +1557,18 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { for (int i = 0; i < maxsplit; i++) { int idx = s.indexOf(delimiter, start); if (idx == -1) break; - parts.add(s.substring(start, idx)); + parts.add(src.substring(start, idx)); start = idx + delimiter.length; } - parts.add(s.substring(start)); + parts.add(src.substring(start)); } else { - parts = s.split(delimiter); + parts = _splitOn(src, delimiter); } } return JinjaList([ for (var i = 0; i < parts.length; i++) - i == parts.length - 1 - ? _derived(obj, parts[i]) - : JinjaStringValue.fromString(parts[i]), + _splitPiece(obj, parts[i], last: i == parts.length - 1), ]); }); case 'rsplit': @@ -1494,13 +1577,14 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { final maxsplit = kwargs['maxsplit']?.asInt ?? (args.length > 1 ? args[1].asInt : -1); - String s = obj.value.toString(); - List parts; + final src = rawOf(obj.value); + final s = src.toString(); + List parts; if (delimiter == null || delimiter == ' ') { // rsplit by whitespace - parts = s.trim().split(RegExp(r'\s+')); + parts = _splitWhitespace(src); if (maxsplit >= 0 && parts.length > maxsplit + 1) { - final rest = parts.sublist(0, parts.length - maxsplit).join(' '); + final rest = _joinSpaces(parts.sublist(0, parts.length - maxsplit)); parts = [rest, ...parts.sublist(parts.length - maxsplit)]; } } else { @@ -1510,26 +1594,28 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { for (int i = 0; i < maxsplit; i++) { int idx = s.lastIndexOf(delimiter, end - 1); if (idx == -1) break; - parts.insert(0, s.substring(idx + delimiter.length, end)); + parts.insert(0, src.substring(idx + delimiter.length, end)); end = idx; } - parts.insert(0, s.substring(0, end)); + parts.insert(0, src.substring(0, end)); } else { - parts = s.split(delimiter); + parts = _splitOn(src, delimiter); } } return JinjaList([ for (var i = 0; i < parts.length; i++) - i == 0 - ? _derived(obj, parts[i]) - : JinjaStringValue.fromString(parts[i]), + _splitPiece(obj, parts[i], last: i == 0), ]); }); case 'capitalize': return JinjaFunction('capitalize', (args, kwargs) { final s = obj.value.toString(); if (s.isEmpty) return obj; - return _retext(obj, s[0].toUpperCase() + s.substring(1).toLowerCase()); + return _retext( + obj, + s[0].toUpperCase() + s.substring(1).toLowerCase(), + () => _capitalized(obj.value), + ); }); case 'title': return JinjaFunction('title', (args, kwargs) { @@ -1543,6 +1629,18 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { return w[0].toUpperCase() + w.substring(1).toLowerCase(); }) .join(' '), + () { + final words = []; + var start = 0; + for (final w in s.split(' ')) { + if (start > 0) words.add(obj.value.substring(start - 1, start)); + words.add( + _capitalized(obj.value.substring(start, start + w.length)), + ); + start += w.length + 1; + } + return joinRaw(words); + }, ); }); case 'format': @@ -1550,67 +1648,105 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { case 'replace': return JinjaFunction('replace', (args, kwargs) { if (args.length < 2) return obj; - final oldVal = args[0].toString(); - final newVal = args[1].toString(); + final newVal = stringOf(args[1]); final count = kwargs['count']?.asInt ?? (args.length > 2 ? args[2].asInt : -1); - - String s = obj.value.toString(); - if (count >= 0) { - String res = s; - int total = 0; - int start = 0; - while (total < count) { - int idx = res.indexOf(oldVal, start); - if (idx == -1) break; - res = res.replaceRange(idx, idx + oldVal.length, newVal); - start = idx + newVal.length; - total++; - } - return _derived(obj, res); - } - - return _derived(obj, s.replaceAll(oldVal, newVal)); + return _derivedFrom( + obj, + _replaceIn(obj.value, args[0].toString(), newVal, count), + [newVal], + ); }); } return null; } +/// `s.trim().split(RegExp(r'\s+'))`, keeping input marking. +List _splitWhitespace(JinjaString s) { + final text = s.toString(); + final trimmed = text.trim(); + final lead = text.length - text.trimLeft().length; + final src = s.substring(lead, lead + trimmed.length); + final pieces = []; + var prev = 0; + for (final m in RegExp(r'\s+').allMatches(trimmed)) { + pieces.add(src.substring(prev, m.start)); + prev = m.end; + } + return pieces..add(src.substring(prev)); +} + +/// `s.split(delimiter)`, keeping input marking. +List _splitOn(JinjaString s, String delimiter) { + final text = s.toString(); + if (delimiter.isEmpty) { + return [for (var i = 0; i < text.length; i++) s.substring(i, i + 1)]; + } + final pieces = []; + var prev = 0; + for (final m in delimiter.allMatches(text)) { + pieces.add(s.substring(prev, m.start)); + prev = m.end; + } + return pieces..add(s.substring(prev)); +} + +/// [pieces] joined with single spaces. +JinjaString _joinSpaces(List pieces) => JinjaString([ + for (var i = 0; i < pieces.length; i++) ...[ + if (i > 0) const JinjaStringPart(' ', false), + ...pieces[i].parts, + ], +]); + +/// A piece of a `split` or `rsplit` of [source]. +/// +/// With input-marked text still to escape, the piece keeps its marking. +/// Otherwise only the [last] piece split off, the remainder, is marked, as +/// in llama.cpp: as input when all of [source] is. +JinjaStringValue _splitPiece( + JinjaStringValue source, + JinjaString piece, { + required bool last, +}) { + if (hasRawInput(source.value)) return JinjaStringValue(joinRaw([piece])); + if (last) return _derived(source, piece.toString()); + return JinjaStringValue.fromString(piece.toString()); +} + +/// [s] with its first character upper case and the rest lower case, changed +/// part by part. +JinjaString _capitalized(JinjaString s) => + joinRaw([s.substring(0, 1).toUpperCase(), s.substring(1).toLowerCase()]); + +/// Formats [args] into the `{}` placeholders of [fmt]. The literal text and +/// each argument keep their input marking. JinjaStringValue _format(JinjaStringValue fmt, List args) { - final literalIsInput = fmt.value.allPartsAreInput; - final source = fmt.value.toString(); + final format = rawOf(fmt.value); + final source = format.toString(); final parts = []; - final literal = StringBuffer(); - void flushLiteral() { - if (literal.isEmpty) return; - parts.add(JinjaStringPart(literal.toString(), literalIsInput)); - literal.clear(); + var literalStart = 0; + void flushLiteral(int end) { + parts.addAll(format.substring(literalStart, end).parts); } var next = 0; for (var i = 0; i < source.length; i++) { - if (source[i] != '{') { - literal.write(source[i]); - continue; - } + if (source[i] != '{') continue; if (i + 1 >= source.length || source[i + 1] != '}') { throw Exception("format() only supports simple '{}' placeholders"); } - i++; if (next >= args.length) { throw Exception( 'format() expected at least ${next + 1} arguments, got ${args.length}', ); } - flushLiteral(); - final arg = args[next++]; - parts.addAll( - arg is JinjaStringValue - ? arg.value.parts - : [JinjaStringPart('$arg', false)], - ); + flushLiteral(i); + i++; + literalStart = i + 1; + parts.addAll(rawOf(stringOf(args[next++])).parts); } - flushLiteral(); + flushLiteral(source.length); return JinjaStringValue(JinjaString(parts)); } @@ -1640,6 +1776,9 @@ JinjaValue _strip(List args, Map kwargs) { if (chars == null) { if (v is JinjaStringValue) return JinjaStringValue(v.value.trim()); + if (hasRawInput(stringOf(v))) { + return JinjaStringValue(_asString(args).value.trim()); + } return JinjaStringValue.fromString(v.toString().trim()); } @@ -1654,9 +1793,14 @@ JinjaValue _strip(List args, Map kwargs) { while (end > start && charSet.contains(s[end - 1])) { end--; } - return JinjaStringValue.fromString(s.substring(start, end)); + return _codeUnits(v, start, end); } +/// Code units [start] to [end] of [v]'s string form, keeping their input +/// marking. +JinjaStringValue _codeUnits(JinjaValue v, int start, [int? end]) => + JinjaStringValue(joinRaw([stringOf(v).substring(start, end)])); + JinjaValue _lstrip(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); final v = args[0]; @@ -1666,6 +1810,9 @@ JinjaValue _lstrip(List args, Map kwargs) { if (chars == null) { if (v is JinjaStringValue) return JinjaStringValue(v.value.trimLeft()); + if (hasRawInput(stringOf(v))) { + return JinjaStringValue(_asString(args).value.trimLeft()); + } return JinjaStringValue.fromString(v.toString().trimLeft()); } @@ -1675,7 +1822,7 @@ JinjaValue _lstrip(List args, Map kwargs) { while (start < s.length && charSet.contains(s[start])) { start++; } - return JinjaStringValue.fromString(s.substring(start)); + return _codeUnits(v, start); } JinjaValue _rstrip(List args, Map kwargs) { @@ -1687,6 +1834,9 @@ JinjaValue _rstrip(List args, Map kwargs) { if (chars == null) { if (v is JinjaStringValue) return JinjaStringValue(v.value.trimRight()); + if (hasRawInput(stringOf(v))) { + return JinjaStringValue(_asString(args).value.trimRight()); + } return JinjaStringValue.fromString(v.toString().trimRight()); } @@ -1696,7 +1846,7 @@ JinjaValue _rstrip(List args, Map kwargs) { while (end > 0 && charSet.contains(s[end - 1])) { end--; } - return JinjaStringValue.fromString(s.substring(0, end)); + return _codeUnits(v, 0, end); } JinjaValue? _resolveNoneMember(JinjaNone obj, String name) { @@ -1716,9 +1866,63 @@ JinjaValue? _resolveNoneMember(JinjaNone obj, String name) { return null; } +/// The members of undefined in llama.cpp, which return an empty value of +/// the filter's result type. +final Map _undefinedMembers = { + for (final name in [ + 'capitalize', + 'join', + 'lower', + 'replace', + 'safe', + 'string', + 'strip', + 'title', + 'truncate', + 'upper', + ]) + name: const JinjaStringValue(JinjaString([])), + for (final name in [ + 'items', + 'list', + 'map', + 'reject', + 'rejectattr', + 'reverse', + 'select', + 'selectattr', + 'sort', + 'unique', + ]) + name: const JinjaList([]), + for (final name in ['length', 'sum', 'wordcount']) + name: const JinjaInteger(0), + for (final name in ['first', 'last', 'max', 'min']) + name: const JinjaUndefined(), +}; + JinjaValue? _resolveUndefinedMember(JinjaUndefined obj, String name) { - // undefined methods mostly return undefined or empty - return JinjaFunction(name, (args, kwargs) => const JinjaUndefined()); + if (name == 'default') { + return _UndefinedMember( + name, + (args, kwargs) => _default([obj, ...args], kwargs), + ); + } + final empty = _undefinedMembers[name]; + if (empty == null) return null; + return _UndefinedMember(name, (args, kwargs) { + // A new list, as a template may append to it. + return empty is JinjaList ? JinjaList([]) : empty; + }); +} + +/// A member of undefined, such as `x.upper` for an undefined `x`. As in +/// llama.cpp, it prints as nothing. +class _UndefinedMember extends JinjaFunction { + const _UndefinedMember(super.name, super.handler); + + @override + String toString() => ''; } JinjaValue _namespace(List args, Map kwargs) { @@ -1799,8 +2003,8 @@ JinjaValue _dictsort(List args, Map kwargs) { JinjaValue _upper(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); final v = args[0]; - if (v is JinjaStringValue) { - return JinjaStringValue(v.value.toUpperCase()); + if (v is JinjaStringValue || hasRawInput(stringOf(v))) { + return JinjaStringValue(_asString(args).value.toUpperCase()); } return JinjaStringValue.fromString(v.toString().toUpperCase()); } @@ -1808,15 +2012,16 @@ JinjaValue _upper(List args, Map kwargs) { JinjaValue _lower(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); final v = args[0]; - if (v is JinjaStringValue) { - return JinjaStringValue(v.value.toLowerCase()); + if (v is JinjaStringValue || hasRawInput(stringOf(v))) { + return JinjaStringValue(_asString(args).value.toLowerCase()); } return JinjaStringValue.fromString(v.toString().toLowerCase()); } JinjaValue _indent(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); - final str = args[0].toString(); + final source = rawOf(stringOf(args[0])); + final str = source.toString(); final width = args.length > 1 ? args[1] : kwargs['width']; final first = args.length > 2 ? args[2].asBool @@ -1826,30 +2031,44 @@ JinjaValue _indent(List args, Map kwargs) { : (kwargs['blank']?.asBool ?? false); final indentStr = width is JinjaStringValue - ? width.toString() - : ' ' * (width?.asInt ?? 4); - final lines = str.isEmpty ? [] : str.split('\n'); + ? width.value + : JinjaString.template(' ' * (width?.asInt ?? 4)); + // Each line as a slice of [source], and the newline after it. + final lines = []; + final newlines = []; + if (str.isNotEmpty) { + var start = 0; + for (final m in '\n'.allMatches(str)) { + lines.add(source.substring(start, m.start)); + newlines.add(source.substring(m.start, m.end)); + start = m.end; + } + lines.add(source.substring(start)); + } final trailingNewline = str.endsWith('\n'); if (trailingNewline) lines.removeLast(); - final buffer = StringBuffer(); + final out = []; for (var i = 0; i < lines.length; i++) { final line = lines[i]; - if (i > 0) buffer.write('\n'); - if (i == 0 ? first : (line.isNotEmpty || blank)) buffer.write(indentStr); - buffer.write(line); + if (i > 0) out.add(newlines[i - 1]); + if (i == 0 ? first : (line.length > 0 || blank)) out.add(indentStr); + out.add(line); } if (trailingNewline) { - buffer.write('\n'); - if (blank) buffer.write(indentStr); + out.add(newlines.last); + if (blank) out.add(indentStr); } - return _derived(args[0], buffer.toString()); + return _derivedFrom( + args[0], + JinjaString([for (final piece in out) ...rawOf(piece).parts]), + [indentStr], + ); } JinjaValue _string(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); - if (args[0] is JinjaStringValue) return args[0]; - return JinjaStringValue.fromString(args[0].toString()); + return _asString(args); } JinjaValue _strftime_now( @@ -1868,6 +2087,10 @@ JinjaValue _strftime_now( if (result.isEmpty || utf8.encode(result).length >= 100) { throw Exception('strftime_now: failed to format time'); } + // Text of an input-marked format may pass through, so it is escaped. + if (hasRawInput(format.value)) { + return JinjaStringValue(JinjaString.user(result)); + } return JinjaStringValue(JinjaString.from(result, isSafe: true)); } @@ -1890,25 +2113,27 @@ JinjaValue _truncate(List args, Map kwargs) { kwargs['length']?.asInt ?? (args.length > 1 ? args[1].asInt : 255); final killwords = kwargs['killwords']?.asBool ?? (args.length > 2 ? args[2].asBool : false); - final end = - kwargs['end']?.toString() ?? - (args.length > 3 ? args[3].toString() : '...'); + final endArg = kwargs['end'] ?? (args.length > 3 ? args[3] : null); + final end = endArg == null ? JinjaString.template('...') : stringOf(endArg); if (s.length <= length) return args[0]; - String res; + int keep; if (killwords) { - res = s.substring(0, length - end.length); + keep = length - end.length; } else { // find last whitespace before length int lastSpace = s.lastIndexOf(' ', length - end.length); if (lastSpace == -1) { - res = s.substring(0, length - end.length); + keep = length - end.length; } else { - res = s.substring(0, lastSpace); + keep = lastSpace; } } - return JinjaStringValue.fromString(res + end); + final kept = s.substring(0, keep); + return JinjaStringValue( + joinRaw([stringOf(args[0]).substring(0, kept.length), end]), + ); } JinjaValue _wordcount(List args, Map kwargs) { diff --git a/lib/src/types/marking.dart b/lib/src/types/marking.dart new file mode 100644 index 0000000..59edf89 --- /dev/null +++ b/lib/src/types/marking.dart @@ -0,0 +1,51 @@ +import 'jinja_string.dart'; +import 'repr.dart'; +import 'value.dart'; + +/// Whether [s] has input-marked text that is escaped on output. +bool hasRawInput(JinjaString s) => !s.isSafe && s.parts.any((p) => p.isInput); + +/// [s] as unescaped text. The text of a safe string is final, so its +/// input-marked parts become template text. +JinjaString rawOf(JinjaString s) { + if (!s.isSafe) return s; + return JinjaString([ + for (final p in s.parts) p.isInput ? JinjaStringPart(p.val, false) : p, + ]); +} + +/// The string form of [v] that keeps the input marking of the strings in it: +/// a string itself, a list, tuple or dict as [reprOf], anything else as +/// template text. +JinjaString stringOf(JinjaValue v) { + if (v is JinjaStringValue) return v.value; + if (v is JinjaList || v is JinjaTuple || v is JinjaMap) return reprOf(v); + return JinjaString.template(v.toString()); +} + +/// Joins [pieces] as unescaped text. +/// +/// Without input-marked text the result is one template part, as a plain +/// string is; otherwise each character keeps the marking of its piece. +JinjaString joinRaw(Iterable pieces) { + final parts = []; + for (final part in [for (final piece in pieces) ...rawOf(piece).parts]) { + if (part.val.isEmpty) continue; + if (parts.isNotEmpty && parts.last.isInput == part.isInput) { + parts.last = JinjaStringPart(parts.last.val + part.val, part.isInput); + } else { + parts.add(part); + } + } + if (!parts.any((p) => p.isInput)) { + return JinjaString.template(parts.map((p) => p.val).join()); + } + return JinjaString(parts); +} + +/// The characters of [s], each keeping its input marking. +List charsOf(JinjaString s) => [ + for (final p in rawOf(s).parts) + for (final c in p.val.split('')) + JinjaStringValue(JinjaString([JinjaStringPart(c, p.isInput)])), +]; diff --git a/test/src/ast/nodes_test.dart b/test/src/ast/nodes_test.dart index f698ade..d6cc919 100644 --- a/test/src/ast/nodes_test.dart +++ b/test/src/ast/nodes_test.dart @@ -833,4 +833,155 @@ void main() { }); } }); + group('Filters on captured output see the input', () { + // A filter block, or a filter on the output of a block set, macro or + // caller(), sees input-marked text as it was passed in, as `x | upper` + // does. Escaping happens once, at output. llama.cpp 7fe450e19 and + // Jinja2 3.1.6 without autoescape give each output with the input + // unescaped; Jinja2 with autoescape filters the escaped text instead. + final values = {'x': JinjaString.user('')}; + const cases = { + '{% filter upper %}{{ x }}{% endfilter %}': '<B>', + '{% set s %}{{ x }}{% endset %}{{ s | upper }}': '<B>', + '{% filter replace("b", "i") %}{{ x }}{% endfilter %}': '<i>', + '{% filter upper %}{% if true %}{{ x }}{% endif %}{% endfilter %}': + '<B>', + '{% macro m() %}{{ x }}{% endmacro %}{{ m() | upper }}': '<B>', + '{% set s %}{% for i in [1] %}{{ x }}{% endfor %}{% endset %}' + '{{ s | replace("<", "[") }}': + '[b>', + '{% filter trim %} {{ x }} {% endfilter %}': '<b>', + '{% filter upper %}{{ x }}{% endfilter %}': '<B>', + '{% set s %}{{ x }}{% endset %}{{ s }}|{{ s ~ x }}': + '<b>|<b><b>', + '{% set s %}a{% endset %}{% filter upper %}{{ s ~ x }}{% endfilter %}': + 'A<B>', + }; + cases.forEach((source, expected) { + test('renders $source', () { + expect(Template(source).render(values), expected); + }); + }); + + // llama.cpp 7fe450e19 output; Jinja2 raises for the filter block and + // counts the escaped text with autoescape. + const lengths = { + '{% filter length %}{{ x }}{% endfilter %}': '3', + '{% set s %}{{ x }}{% endset %}{{ s | length }}': '3', + '{% macro m() %}{{ caller() | length }}{% endmacro %}' + '{% call m() %}{{ x }}{% endcall %}': + '3', + }; + lengths.forEach((source, expected) { + test('renders $source', () { + expect(Template(source).render(values), expected); + }); + }); + + test('safe turns escaping off, as for {{ x | safe }}', () { + expect( + Template( + '{% filter safe %}{{ x }}{% endfilter %}|' + '{% set s %}{{ x }}{% endset %}{{ s | safe }}', + ).render(values), + '|', + ); + }); + + test('escapes input next to safe input before filtering', () { + // The safe text is final, so the rest is escaped first, as with `~`. + expect( + Template( + '{% filter upper %}{{ x }}{{ x | safe }}{% endfilter %}', + ).render(values), + '<B>', + ); + }); + + test('keeps joining safe and unsafe template text into one part', () { + final result = Template( + '{{ "a" ~ ([1] | tojson) }}', + ).renderJinjaResult({}); + expect(result.parts.map((p) => (p.val, p.isInput)), [('a[1]', false)]); + }); + }); + + group('Member access on undefined', () { + // llama.cpp 7fe450e19 output. Jinja2 3.1.6 raises UndefinedError. + const cases = { + '[{{ y.text }}]': '[]', + "[{{ y[0]['text'] }}]": '[]', + '[{{ y[0].text }}]': '[]', + '[{{ y.foo }}]|[{{ y[0].foo }}]': '[]|[]', + '[{{ y[0].upper }}]|[{{ y[0].items }}]': '[]|[]', + "{{ '[' ~ y[0]['text'] ~ ']' }}": '[]', + "[{{ (y[0]['text']) is defined }}]|[{{ y[0] is defined }}]": + '[False]|[False]', + "[{{ y.upper is defined }}]|[{{ y.text is defined }}]|[{{ y['upper'] is defined }}]|[{{ y.get is defined }}]": + '[True]|[False]|[False]|[False]', + '[{{ y.upper() }}]|[{{ y.length() }}]|[{{ y.items() | length }}]|[{{ y.default(1) }}]|[{{ y.first() is defined }}]|[{{ y.sum() }}]': + '[]|[0]|[0]|[1]|[False]|[0]', + '[{{ y[none] }}]|[{{ y.text.more }}]|[{{ y[0][1][2] }}]': '[]|[]|[]', + "{% for m in y.items() %}x{% endfor %}|{% for c in y[0]['text'] %}x{% endfor %}": + '|', + }; + cases.forEach((source, expected) { + test('renders $source', () { + expect(Template(source).render(), expected); + }); + }); + + test('a member function it lacks cannot be called', () { + // llama.cpp: "Callee is not a function"; Jinja2 raises UndefinedError. + expect(() => Template('{{ y.foo() }}').render(), throwsException); + }); + + test('a missing message content prints nothing', () { + // Devstral Small 2507 prints message['content'][0]['text'] when the + // content is not a string. llama.cpp 7fe450e19 output; Jinja2 raises. + const source = "[{{ m['content'][0]['text'] }}]"; + expect( + Template(source).render({ + 'm': {'role': 'assistant'}, + }), + '[]', + ); + expect( + Template(source).render({ + 'm': {'role': 'assistant', 'content': ''}, + }), + '[]', + ); + }); + }); + + group('Subscript of none', () { + test('throws for a number, as in llama.cpp', () { + // llama.cpp 7fe450e19 raises this message; Jinja2 3.1.6 raises + // "None has no element 0" for x[0]['text'] and prints x[0] as ''. + for (final source in ["{{ x[0]['text'] }}", '{{ x[0] }}']) { + expect( + () => Template(source).render({'x': null}), + throwsA( + predicate( + (e) => e.toString().contains( + 'Cannot access property with non-string: got Integer', + ), + ), + ), + reason: source, + ); + } + }); + + test('gives undefined for a name', () { + // llama.cpp 7fe450e19 and Jinja2 3.1.6 output. + expect( + Template( + "[{{ x['text'] }}]|[{{ x.text }}]|[{{ x.text is defined }}]", + ).render({'x': null}), + '[]|[]|[False]', + ); + }); + }); } diff --git a/test/src/input_escaping_test.dart b/test/src/input_escaping_test.dart new file mode 100644 index 0000000..c00313b --- /dev/null +++ b/test/src/input_escaping_test.dart @@ -0,0 +1,215 @@ +import 'package:dinja/dinja.dart'; +import 'package:test/test.dart'; + +void main() { + group('Input-marked text keeps its marking', () { + // llama.cpp 7fe450e19 drops the marking in these, or lacks the filter; + // it gives each output with the input unescaped. dinja escapes it. + final values = {'x': JinjaString.user('')}; + const cases = { + '{{ x[0] }}|{{ x[::2] }}': '<|<>', + '{{ x | first }}|{{ x | last }}|{{ x | reverse }}': '<|>|>b<', + '{{ x | list | join }}|{% for c in x %}{{ c }}{% endfor %}|{{ x | unique | join }}': + '<b>|<b>|<b>', + "{{ x.split('b') | first }}|{{ x.rsplit('b') | last }}|{{ x.split() | join }}": + '<|>|<b>', + "{{ x.strip('<') }}|{{ x.lstrip('<') }}|{{ x.rstrip('>') }}": + 'b>|b>|<b', + '{{ x | truncate(2, true, "") }}|{{ "abcdef" | truncate(5, true, x) }}': + '<b|ab<b>', + '{{ str(x) }}|{{ [x] | string }}|{{ "a" ~ [x] }}|{{ "{}" | format([x]) }}': + "<b>|['<b>']|a['<b>']|['<b>']", + '{{ ("a" ~ x) | indent(2, true) }}|{{ ("a" ~ x) | replace("a", "z") }}|{{ "a" | replace("a", x) }}': + ' a<b>|z<b>|<b>', + '{{ ("a" ~ x).format() }}|{{ [x] | upper }}|{{ [x] | trim }}': + "a<b>|['<B>']|['<b>']", + '{{ (x ~ "a") | capitalize }}': '<b>a', + }; + cases.forEach((source, expected) { + test('renders $source', () { + expect(Template(source).render(values), expected); + }); + }); + + test('in a case change that can change the length', () { + // On the web 'ß' upper-cases to 'SS'; on the VM it stays 'ß'. + final out = Template( + '{{ (x ~ "a") | capitalize }}|{{ (x ~ " a") | title }}', + ).render({'x': JinjaString.user('ß<')}); + expect(out, matches(RegExp(r'^(SS|ß)<a\|(SS|ß)< A$'))); + }); + + test('in a dict key', () { + expect( + Template( + '{% for k in d %}{{ k }}{% endfor %}|{{ d | list | join }}', + ).render({ + 'd': {JinjaString.user(''): 1}, + }), + '<k>|<k>', + ); + }); + + test('in a strftime_now format', () { + expect( + Template( + '{{ strftime_now(x) }}', + ).render({'x': JinjaString.user('<%Y')}), + startsWith('<'), + ); + }); + }); + + group('Template text is not escaped', () { + // llama.cpp 7fe450e19 marks all of a replace or indent result as input + // when all of its source is. + final values = {'x': JinjaString.user('')}; + const cases = { + '{{ x | replace("b", "") }}': '<>', + '{{ x.replace("b", "", 1) }}': '<>', + '{{ x | indent("> ", true) }}': '> <b>', + }; + cases.forEach((source, expected) { + test('renders $source', () { + expect(Template(source).render(values), expected); + }); + }); + }); + + group('Safe text is not escaped again', () { + final values = {'x': JinjaString.user(''), 'y': JinjaString.user('')}; + const cases = { + '{{ (x ~ (y | safe)).format() }}': '<b>', + '{{ "{}" | format(x ~ (y | safe)) }}': '<b>', + '{{ (x ~ (y | safe)) | list | join }}': '<b>', + '{{ "a" | indent(x ~ (y | safe), true) }}': '<b>a', + '{{ (x | safe)[::2] }}': '<>', + }; + cases.forEach((source, expected) { + test('renders $source', () { + expect(Template(source).render(values), expected); + }); + }); + }); + + group('Escaping invariants', () { + // Every expression in every wrapper: input-marked text is escaped + // exactly once, and template text is never escaped. + const expressions = [ + 'x', + 'x | upper', + 'x | title', + 'x | capitalize', + 'x | trim', + 'x.strip("<")', + 'x | replace("b", "[T]")', + 'x.replace("<", "[")', + 'x | indent(2, true)', + 'x[0]', + 'x[1:]', + 'x[::-1]', + 'x | first', + 'x | reverse', + 'x | list | join', + 'x | join(",")', + '[x, "[T]"] | join(x)', + 'l | map("upper") | join', + 'x | tojson', + '{x: [x]} | tojson(indent=1)', + 'd | tojson', + 'd | items | list', + 'd | list | join', + 'x.split("b") | join("|")', + 'x.rsplit("b", 1) | first', + '"abcdef" | truncate(5, true, x)', + 'x | string', + 'str([x])', + 'x ~ "[T]"', + '"[T]" ~ [x]', + 'x * 2', + '"{}[T]".format(x)', + '"{}" | format(d)', + '("a" ~ x) | replace("a", x)', + 'x | default("d")', + 'nope | default(x)', + 'x if true else ""', + '{"k": x}["k"]', + 'namespace(a=x).a', + 'l | sort | join', + 'l | unique | join', + 'l | max', + 'x | length', + ]; + const wrappers = [ + '{{ E }}', + '{% if true %}{{ E }}{% endif %}', + '{% for i in [1] %}{{ E }}{% endfor %}', + '{% for c in [E] %}{{ c }}{% endfor %}', + '{% set s = E %}{{ s }}', + '{% set s %}{{ E }}{% endset %}{{ s }}', + '{% set s %}{{ E }}[T]{% endset %}{{ s | upper }}{{ s | length }}', + '{% macro m() %}{{ E }}{% endmacro %}{{ m() }}', + '{% macro m(a) %}[{{ a }}]{% endmacro %}{{ m(E) }}', + '{% macro m() %}{{ caller() | trim }}{% endmacro %}' + '{% call m() %}{{ E }}{% endcall %}', + '{% filter upper %}{{ E }}{% endfilter %}', + '{% filter replace("b", "[") %}{{ E }}{% endfilter %}', + '{% filter tojson %}{{ E }}{% endfilter %}', + '{{ (E) | string }}', + '{{ [E] | join }}', + '{{ "p" ~ (E) ~ "q" }}', + '{% for i in [1, 2] %}{{ E }}{% if loop.first %}{% continue %}' + '{% endif %}{% endfor %}', + ]; + final entity = RegExp(r'&(lt|gt|amp|quot|#39);'); + + test('escape input-marked text once', () { + // Template text here has none of < > &, so each one in the output + // must come from escaping. + final values = { + 'x': JinjaString.user('&'), + 'l': [JinjaString.user('

'), JinjaString.user('q&')], + 'd': { + JinjaString.user(''): [ + JinjaString.user(''), + {'n': JinjaString.user('&')}, + ], + }, + }; + for (final w in wrappers) { + for (final e in expressions) { + final source = w.replaceAll('E', e); + final out = Template(source).render(values); + final stripped = out.replaceAll(entity, ''); + expect(stripped, isNot(matches(r'[<>&]')), reason: source); + expect(out, isNot(contains('&lt;')), reason: source); + expect(out, isNot(contains('&amp;')), reason: source); + } + } + }); + + test('never escape template text', () { + // Input here has none of < > & ' ", so the output has no entity. + final values = { + 'x': JinjaString.user('abc'), + 'l': [JinjaString.user('p'), JinjaString.user('q')], + 'd': { + JinjaString.user('k'): [ + JinjaString.user('v'), + {'n': JinjaString.user('w')}, + ], + }, + }; + for (final w in wrappers) { + for (final e in expressions) { + final source = w + .replaceAll('E', e) + .replaceAll('[T]', "") + .replaceAll('"b"', '"a"'); + final out = Template(source).render(values); + expect(out, isNot(contains(entity)), reason: source); + } + } + }); + }); +} diff --git a/test/src/runtime/builtins_test.dart b/test/src/runtime/builtins_test.dart index ebda4d2..37f7976 100644 --- a/test/src/runtime/builtins_test.dart +++ b/test/src/runtime/builtins_test.dart @@ -920,18 +920,6 @@ void main() { }, '[\n {\n "type": "function",\n "function": {\n "name": "get_weather",\n "description": "Get the weather in a city, e.g. \\"Montréal\\" \'Paris\' & more",\n "parameters": {\n "type": "object",\n "properties": {\n "city": {\n "type": "string",\n "description": "City name"\n },\n "days": {\n "type": "integer",\n "minimum": 1,\n "maximum": 7,\n "default": 1.5\n }\n },\n "required": [\n "city"\n ]\n }\n }\n }\n]', ), - ( - 'input-marked strings are not escaped', - '{{ x | tojson }}', - { - 'x': { - 't': [ - {'d': JinjaString.user('')}, - ], - }, - }, - '{"t": [{"d": ""}]}', - ), ]; for (final (name, source, data, expected) in cases) { test(name, () => expect(Template(source).render(data), expected)); @@ -1293,4 +1281,153 @@ void main() { }); }); }); + group('tojson keeps input marking', () { + // llama.cpp 7fe450e19 gives each output with the input unescaped, as + // its tojson drops the marking and it never escapes. Only the input + // text is escaped, after JSON escaping; the JSON's own quotes are not. + final x = JinjaString.user(''); + final cases = <(String, Map, String)>[ + ('{{ x | tojson }}', {'x': x}, '"<b>"'), + ('{{ {"k": x} | tojson }}', {'x': x}, '{"k": "<b>"}'), + ( + '{{ d | tojson }}', + { + 'd': {JinjaString.user(''): 1}, + }, + '{"<k>": 1}', + ), + ( + '{{ d | tojson }}', + { + 'd': { + 'a': [ + x, + {'c': x}, + ], + }, + }, + '{"a": ["<b>", {"c": "<b>"}]}', + ), + ( + '{{ x | tojson }}', + {'x': JinjaString.user('"q"')}, + r'"\"q\""', + ), + ( + '{{ x | tojson(ensure_ascii=true) }}', + {'x': JinjaString.user('é<')}, + r'"\u00e9<"', + ), + ( + '{{ {"k": x} | tojson(indent=2) }}', + {'x': x}, + '{\n "k": "<b>"\n}', + ), + ( + '{{ [x, 1] | tojson(separators=(sep, ":")) }}', + {'x': x, 'sep': JinjaString.user(';<')}, + '["<b>";<1]', + ), + ( + '{{ [x, 1] | tojson(separators=("<,>", ":")) }}', + {'x': x}, + '["<b>"<,>1]', + ), + // llama.cpp does not implement sort_keys. + ( + '{{ d | tojson(sort_keys=true) }}', + { + 'd': {'b': 1, JinjaString.user(''): x}, + }, + '{"<a>": "<b>", "b": 1}', + ), + ('{{ x | tojson | safe }}', {'x': x}, '""'), + ]; + for (final (source, data, expected) in cases) { + test('renders $source', () { + expect(Template(source).render(data), expected); + }); + } + + test('marks only the input text', () { + final result = Template( + '{{ {"k": x} | tojson }}', + ).renderJinjaResult({'x': x}); + expect(result.parts, [ + const JinjaStringPart('{"k": "', false), + const JinjaStringPart('<b>', true), + const JinjaStringPart('"}', false), + ]); + }); + + test('is safe template text without input', () { + const source = '{{ d | tojson }}|{{ d | tojson is escaped }}'; + final data = { + 'd': {'': ''}, + }; + expect(Template(source).render(data), '{"": ""}|True'); + }); + }); + + group('join keeps input marking', () { + // llama.cpp 7fe450e19 gives each output with the input unescaped, as + // its join drops the marking and it never escapes. It does not map with + // a filter, join a string, or join none or a list. + final x = JinjaString.user(''); + final cases = <(String, Map, String)>[ + ('{{ [x, ""] | join }}', {'x': x}, '<b>'), + ( + '{{ [x, x] | join(sep) }}', + {'x': x, 'sep': JinjaString.user('<,>')}, + '<b><,><b>', + ), + ('{{ [x, x] | join("<,>") }}', {'x': x}, '<b><,><b>'), + ( + '{{ l | map("upper") | join }}', + { + 'l': [JinjaString.user(''), x], + }, + '<A><B>', + ), + ( + '{{ d | map(attribute="n") | join(", ") }}', + { + 'd': [ + {'n': JinjaString.user('')}, + {'n': x}, + ], + }, + '<a>, <b>', + ), + ( + '{{ d | join(", ", attribute="n") }}', + { + 'd': [ + {'n': x}, + ], + }, + '<b>', + ), + ('{{ [x, 1, none] | join }}', {'x': x}, '<b>1None'), + ('{{ [[x]] | join }}', {'x': x}, "['<b>']"), + ('{{ x | join }}', {'x': x}, '<b>'), + ('{{ [x | safe] | join }}', {'x': x}, ''), + ]; + for (final (source, data, expected) in cases) { + test('renders $source', () { + expect(Template(source).render(data), expected); + }); + } + + test('marks only the input text', () { + final result = Template( + '{{ [x, x] | join(", ") }}', + ).renderJinjaResult({'x': x}); + expect(result.parts, [ + const JinjaStringPart('<b>', true), + const JinjaStringPart(', ', false), + const JinjaStringPart('<b>', true), + ]); + }); + }); } diff --git a/test/src/types/jinja_string_test.dart b/test/src/types/jinja_string_test.dart index a80c348..2552602 100644 --- a/test/src/types/jinja_string_test.dart +++ b/test/src/types/jinja_string_test.dart @@ -266,7 +266,7 @@ void main() { }); group('Input marking matches llama.cpp', () { - // Parts as llama.cpp e85e15cf6 marks them, adjacent parts merged: + // Parts as llama.cpp 7fe450e19 marks them, adjacent parts merged: // [I:...] is input, [T:...] is template text. for (final (source, expected) in [ ('{{ s ~ t }}', '[I: Ab,c xy]'), @@ -276,9 +276,7 @@ void main() { ('{{ (s ~ t) | upper }}', '[I: AB,C XY]'), ('{{ s * 2 }}', '[I: Ab,c Ab,c ]'), ("{{ ('lit' ~ s) * 2 }}", '[T:lit][I: Ab,c ][T:lit][I: Ab,c ]'), - ("{{ s | replace('A', 'Z') }}", '[I: Zb,c ]'), ("{{ s | replace('A', t) }}", '[I: xyb,c ]'), - ("{{ s.replace('A', 'Z') }}", '[I: Zb,c ]'), ('{{ s | capitalize }}', '[I: ab,c ]'), ('{{ s.capitalize() }}', '[I: ab,c ]'), ("{{ ('lit' ~ s) | capitalize }}", '[T:Lit][I: ab,c ]'), @@ -286,18 +284,10 @@ void main() { ('{{ s | title }}', '[I: Ab,c ]'), ('{{ s.title() }}', '[I: Ab,c ]'), ('{{ s | string }}', '[I: Ab,c ]'), - ('{{ s | indent(2, true) }}', '[I: Ab,c ]'), ("{{ s.split(',') | last }}", '[I:c ]'), ("{{ s.split(',', 1) | last }}", '[I:c ]'), ("{{ t.split(',') | first }}", '[I:xy]'), ("{{ s.rsplit(',') | first }}", '[I: Ab]'), - // llama.cpp drops the marking in these. - ('{{ l | join }}', '[T:pq]'), - ("{{ s.split(',') | first }}", '[T: Ab]'), - ("{{ s.rsplit(',') | last }}", '[T:c ]'), - ("{{ ('lit' ~ s) | replace('A', 'Z') }}", '[T:lit Zb,c ]'), - ("{{ ('lit' ~ s) | indent(2) }}", '[T:lit Ab,c ]'), - ('{{ s[1] }}', '[T:A]'), ('{{ s + t }}', '[I: Ab,c xy]'), ]) { test(source, () { @@ -306,6 +296,42 @@ void main() { } }); + group('Input marking kept where llama.cpp drops or widens it', () { + // Each character keeps the marking of where it came from, so input text + // is escaped and template text is not. llama.cpp 7fe450e19 marks each as + // noted; its marking only controls special-token parsing. + for (final (source, expected) in [ + // llama.cpp: [T:pq] + ('{{ l | join }}', '[I:pq]'), + // llama.cpp: [T:p, q] + ("{{ l | join(', ') }}", '[I:p][T:, ][I:q]'), + // llama.cpp: [T:" Ab,c "] + ('{{ s | tojson }}', '[T:"][I: Ab,c ][T:"]'), + // llama.cpp: [T:{" Ab,c ": "xy"}] + ('{{ {s: t} | tojson }}', '[T:{"][I: Ab,c ][T:": "][I:xy][T:"}]'), + // llama.cpp: [T: Ab] + ("{{ s.split(',') | first }}", '[I: Ab]'), + // llama.cpp: [T:c ] + ("{{ s.rsplit(',') | last }}", '[I:c ]'), + // llama.cpp: [T:A] + ('{{ s[1] }}', '[I:A]'), + // llama.cpp: [T:lit Zb,c ] + ("{{ ('lit' ~ s) | replace('A', 'Z') }}", '[T:lit][I: ][T:Z][I:b,c ]'), + // llama.cpp: [T:lit Ab,c ] + ("{{ ('lit' ~ s) | indent(2) }}", '[T:lit][I: Ab,c ]'), + // llama.cpp: [I: Zb,c ] + ("{{ s | replace('A', 'Z') }}", '[I: ][T:Z][I:b,c ]'), + // llama.cpp: [I: Zb,c ] + ("{{ s.replace('A', 'Z') }}", '[I: ][T:Z][I:b,c ]'), + // llama.cpp: [I: Ab,c ] + ('{{ s | indent(2, true) }}', '[T: ][I: Ab,c ]'), + ]) { + test(source, () { + expect(_marks(source), expected); + }); + } + }); + group('Input escaping', () { for (final (source, expected) in [ ("{{ s ~ '' }}", 'a<b'), From 12e9bf4acd4257efbd06eb6b188cf429c29ac322 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Sat, 26 Sep 2026 10:46:28 -0400 Subject: [PATCH 2/3] fix: escape input when safe is applied to rendered output; keep format parts --- CHANGELOG.md | 4 +- README.md | 2 +- lib/src/ast/nodes.dart | 69 +++++++++++++++---- lib/src/runtime/builtins.dart | 16 ++++- lib/src/types/marking.dart | 74 ++++++++++++++++++++ test/src/ast/nodes_test.dart | 102 +++++++++++++++++++++++++--- test/src/input_escaping_test.dart | 24 +++++++ test/src/runtime/builtins_test.dart | 5 ++ 8 files changed, 267 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d5a4e7..685eea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,11 @@ ## Unreleased -- Fixed a `{% filter %}` block, and a filter on the output of a block `set`, a macro or `caller()`, getting `JinjaString.user` text already escaped: with `x = JinjaString.user('')`, `{% filter upper %}{{ x }}{% endfilter %}` is now `<B>`, not `<B>`, and `{% filter length %}{{ x }}{% endfilter %}` is `3`, not `9`, as in llama.cpp. Input text is now escaped once, when the template outputs it. So `| safe` on such output, and `{% filter safe %}`, now turn escaping off for the input in it, as `{{ x | safe }}` does. A block whose output also contains `| safe` input text is still escaped before the filter runs, as `~` does. +- Fixed a `{% filter %}` block, and a filter on the output of a block `set`, a macro or `caller()`, getting `JinjaString.user` text already escaped: with `x = JinjaString.user('')`, `{% filter upper %}{{ x }}{% endfilter %}` is now `<B>`, not `<B>`, and `{% filter length %}{{ x }}{% endfilter %}` is `3`, not `9`, as in llama.cpp. Input text is now escaped once, when the template outputs it. `| safe` on such output, `{% filter safe %}`, and `| safe` on text made from it or from other safe text escape the input in it, as Jinja2 with autoescape does; `{{ x | safe }}` still outputs `x` as is. A block whose output also contains `| safe` input text is still escaped before the filter runs, as `~` does. - Fixed `tojson` and `join` dropping the input marking, so `JinjaString.user` text in their output was not escaped: `{{ {"k": x} | tojson }}` is now `{"k": "<b>"}` and `{{ [x, ""] | join }}` is `<b>`. Only input text is escaped, after JSON escaping, in values, dict keys and separators; the JSON's own quotes and template text are not. `tojson` output with input text in it is no longer safe (`is escaped`). - Fixed input text being output unescaped by string indexing, `first`, `last`, `reverse`, `list` and `for` on a string, `split` and `rsplit` pieces, `strip`, `lstrip` and `rstrip` with characters, `truncate`, `str()`, `~`, `+` and `format` with a list or dict, `string`, `upper`, `lower`, `title`, `capitalize`, `trim`, `replace` and `indent` on a list or dict, `replace`, `indent` and `format` on text that is only partly input, input passed as a `replace` replacement, input dict keys in a `for` loop, and an input `strftime_now` format. - Changed `replace` and `indent` on input text to leave the template text they add unescaped: `x | replace('b', '')` is now `<>`, not `<<i>>`, and `x | indent('> ', true)` no longer escapes `> `. Each character keeps the marking of the text it came from; llama.cpp marks all of the result as input when all of the source is. - Fixed `| safe` text being escaped when passed to `format`, or to `replace` or `indent` on input text, sliced with a step or split by `unique`, and escaped twice when it held escaped input: `'{}'.format(x ~ (y | safe))` is now `<b>`, not `&lt;b&gt;`. -- Fixed member access on an undefined value giving a function, which printed as `` and was `defined`: for an undefined `y`, `{{ y.text }}` and `{{ y[0]['text'] }}` now print nothing and are not `defined`, as in llama.cpp. As in llama.cpp, `y.upper`, `y.length` and the other filters that llama.cpp gives undefined are functions returning an empty value (`y.length()` is `0`, `y.default(1)` is `1`) that print nothing, and calling any other member, such as `y.get()`, throws. Devstral Small 2507 no longer prints `` for an assistant turn without `content`. +- Fixed member access on an undefined value giving a function, which printed as `` and was `defined`: for an undefined `y`, `{{ y.text }}` and `{{ y[0]['text'] }}` now print nothing and are not `defined`, as in llama.cpp. As in llama.cpp, `y.upper`, `y.length` and the other filters that llama.cpp gives undefined are functions returning an empty value (`y.length()` is `0`, `y.default(1)` is `1`) that print nothing, and calling any other member, such as `y.get()`, throws. Devstral Small 2507 no longer prints `` for an assistant turn without `content`. Also as in llama.cpp, Command R+ tool use renders an `object` parameter without `additionalProperties` as `Dict[str, Union[]]`, not `Dict[str, Any]`, and QwQ-32B throws for an assistant tool-call turn without `content` before the last turn. - Changed a number subscript of none, as in `x[0]` or `x[0]['text']` for `x = none`, to throw `Cannot access property with non-string: got Integer`, as llama.cpp does; it gave undefined. Devstral Small 2507 with an assistant turn whose content is none now throws, as in llama.cpp. ## 1.2.0 diff --git a/README.md b/README.md index add8893..bbe9ac6 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Real templates come from the `chat_template` field of a model's `tokenizer_confi Wrap untrusted values in `JinjaString.user`. They are escaped when rendered; a plain `String` is treated as template text and is not. -Input text keeps its marking through filters, `~`, loops, macros and `{% set %}` and `{% filter %}` blocks, and is escaped once, when the template outputs it. So a filter sees it as it was passed in: `{% filter length %}{{ name }}{% endfilter %}` counts the characters of `name`, not of its escaped form. `tojson` and `join` escape only the input text in their output, not the JSON's quotes or the template's separator. `| safe` and `markSafe()` turn escaping off. +Input text keeps its marking through filters, `~`, loops, macros and `{% set %}` and `{% filter %}` blocks, and is escaped once, when the template outputs it. So a filter sees it as it was passed in: `{% filter length %}{{ name }}{% endfilter %}` counts the characters of `name`, not of its escaped form. `tojson` and `join` escape only the input text in their output, not the JSON's quotes or the template's separator. `| safe` and `markSafe()` turn escaping off for a value. The output of a block `set`, macro, `caller()` or `{% filter %}` block is final, as in Jinja2 with autoescape, so `| safe` on it escapes the input in it. ```dart import 'package:dinja/dinja.dart'; diff --git a/lib/src/ast/nodes.dart b/lib/src/ast/nodes.dart index edf328f..2a9f108 100644 --- a/lib/src/ast/nodes.dart +++ b/lib/src/ast/nodes.dart @@ -52,8 +52,11 @@ class Program extends Statement { /// Renders the template. Input-marked text is escaped here, once. @override JinjaValue execute(Context ctx) { - final result = execStatements(body, ctx) as JinjaStringValue; - return JinjaStringValue(result.value.escape()); + final result = renderScope( + ctx.environment.values, + () => execStatements(body, ctx), + ); + return JinjaStringValue((result as JinjaStringValue).value.escape()); } } @@ -103,7 +106,9 @@ JinjaString _joinOutput(List pieces) { /// `continue` leaving the body discards the captured output, as in llama.cpp. JinjaValue _execCaptured(List stmts, Context ctx) { try { - return execStatements(stmts, ctx); + final result = execStatements(stmts, ctx) as JinjaStringValue; + markRendered(result.value); + return result; } on LoopSignal catch (signal) { signal.output.clear(); rethrow; @@ -211,6 +216,7 @@ class ForStatement extends Statement { items = iterableVal.items; } else if (iterableVal is JinjaStringValue) { items = charsOf(iterableVal.value); + deriveRendered(JinjaList(items), [iterableVal]); } // Filter items first if testExpr @@ -715,9 +721,27 @@ class MemberExpression extends Expression { @override String get type => 'MemberExpression'; - @override @override JinjaValue execute(Context ctx) { + JinjaValue? obj; + final result = _access(ctx, (value) => obj = value); + final source = obj; + if (source == null) return result; + if (result is JinjaFunction) { + if (!holdsFinal(source)) return result; + return JinjaFunction( + result.name, + (args, kwargs) => deriveRendered(result.handler(args, kwargs), [ + source, + ...args, + ...kwargs.values, + ]), + ); + } + return deriveRendered(result, [source]); + } + + JinjaValue _access(Context ctx, void Function(JinjaValue) onObject) { // Handle slice expressions first, as they are a special case of computed access if (computed && property is SliceExpression) { final slice = property as SliceExpression; @@ -726,6 +750,7 @@ class MemberExpression extends Expression { final stepVal = slice.step?.execute(ctx); final obj = object.execute(ctx); + onObject(obj); int? start = (startVal != null && !startVal.isNone) ? startVal.asInt @@ -810,6 +835,7 @@ class MemberExpression extends Expression { if (computed || property is IntegerLiteral) { // Bracket notation: obj[expr], or an index after a dot: obj.0 final obj = object.execute(ctx); + onObject(obj); final prop = property.execute(ctx); if (!computed && prop.asInt < 0) { throw Exception('Static member property cannot be negative'); @@ -879,6 +905,7 @@ class MemberExpression extends Expression { } final propName = (property as Identifier).name; final obj = object.execute(ctx); + onObject(obj); // Priority: 1. Attributes/Methods, 2. Map keys, 3. Special cases (length) final method = resolveMember(obj, propName); @@ -952,7 +979,10 @@ class CallExpression extends Expression { } } - return func.handler(positionals, kwargs); + return deriveRendered(func.handler(positionals, kwargs), [ + ...positionals, + ...kwargs.values, + ]); } } @@ -1014,17 +1044,23 @@ class BinaryExpression extends Expression { } // String repeat? `~` is concat. `*` is repeat in Python. if (l is JinjaStringValue && r.isNumeric) { - return JinjaStringValue( - JinjaString([ - for (var i = 0; i < r.asInt; i++) ...l.value.parts, - ], isSafe: l.isSafe), + return deriveRendered( + JinjaStringValue( + JinjaString([ + for (var i = 0; i < r.asInt; i++) ...l.value.parts, + ], isSafe: l.isSafe), + ), + [l], ); } if (l is JinjaInteger && r is JinjaStringValue) { - return JinjaStringValue( - JinjaString([ - for (var i = 0; i < l.value; i++) ...r.value.parts, - ], isSafe: r.isSafe), + return deriveRendered( + JinjaStringValue( + JinjaString([ + for (var i = 0; i < l.value; i++) ...r.value.parts, + ], isSafe: r.isSafe), + ), + [r], ); } throw Exception('Invalid operand types for *'); @@ -1194,7 +1230,7 @@ class FilterExpression extends Expression { if (globalFilters.containsKey(filterName)) { final res = globalFilters[filterName]!(args, kwargs); - return res; + return deriveRendered(res, [...args, ...kwargs.values]); } throw Exception('Unknown filter: $filterName'); } @@ -1250,7 +1286,10 @@ class TestExpression extends Expression { } } -JinjaStringValue _concat(JinjaValue l, JinjaValue r) { +JinjaValue _concat(JinjaValue l, JinjaValue r) => + deriveRendered(_join(l, r), [l, r]); + +JinjaStringValue _join(JinjaValue l, JinjaValue r) { final left = _operand(l); final right = _operand(r); if (left.isSafe == right.isSafe) return JinjaStringValue(left + right); diff --git a/lib/src/runtime/builtins.dart b/lib/src/runtime/builtins.dart index 56d3e91..15ef5b8 100644 --- a/lib/src/runtime/builtins.dart +++ b/lib/src/runtime/builtins.dart @@ -212,7 +212,7 @@ JinjaStringValue _asString(List args) { if (v is JinjaStringValue) return v; final s = stringOf(v); return hasRawInput(s) - ? JinjaStringValue(s) + ? JinjaStringValue(joinRaw([s])) : JinjaStringValue.fromString(v.toString()); } @@ -1089,6 +1089,7 @@ JinjaValue _safe(List args, Map kwargs) { } final v = args[0]; if (v is JinjaStringValue) { + if (isRendered(v.value)) return JinjaStringValue(v.value.escape()); return JinjaStringValue(v.value.markSafe()); } return JinjaStringValue(JinjaString.from(v.toString(), isSafe: true)); @@ -1364,7 +1365,8 @@ JinjaValue _testIsEscaped( Map kwargs, ) { if (args.isEmpty) return const JinjaBoolean(false); - return JinjaBoolean(args[0].isSafe); + final v = args[0]; + return JinjaBoolean(v.isSafe || v is JinjaStringValue && isRendered(v.value)); } JinjaValue _testIsFilter( @@ -1744,7 +1746,15 @@ JinjaStringValue _format(JinjaStringValue fmt, List args) { flushLiteral(i); i++; literalStart = i + 1; - parts.addAll(rawOf(stringOf(args[next++])).parts); + final arg = args[next++]; + final text = stringOf(arg); + parts.addAll( + arg is JinjaStringValue + ? rawOf(text).parts + : hasRawInput(text) + ? joinRaw([text]).parts + : [JinjaStringPart('$arg', false)], + ); } flushLiteral(source.length); return JinjaStringValue(JinjaString(parts)); diff --git a/lib/src/types/marking.dart b/lib/src/types/marking.dart index 59edf89..e2144a2 100644 --- a/lib/src/types/marking.dart +++ b/lib/src/types/marking.dart @@ -49,3 +49,77 @@ List charsOf(JinjaString s) => [ for (final c in p.val.split('')) JinjaStringValue(JinjaString([JinjaStringPart(c, p.isInput)])), ]; + +final Expando _rendered = Expando(); +var _inputSeen = false; + +/// Runs [render] over the [values] it is given. Rendered output is tracked +/// only when they hold input-marked text. +T renderScope(Iterable values, T Function() render) { + final saved = _inputSeen; + var input = false; + _walk(values, (s) => input = input || hasRawInput(s)); + _inputSeen = input; + try { + return render(); + } finally { + _inputSeen = saved; + } +} + +/// Records [s] as rendered output: a block `set`, macro, `caller()` or +/// `filter` block body. Its text is final, as in Jinja2 with autoescape, so +/// `safe` escapes the input-marked text in it. +void markRendered(JinjaString s) { + if (hasRawInput(s)) _rendered[s] = true; +} + +/// Whether [s] is rendered output, or derived from it or from safe text. +bool isRendered(JinjaString s) => _rendered[s] ?? false; + +bool _isFinal(JinjaString s) => s.isSafe || isRendered(s); + +/// Whether [v] is or holds safe text or rendered output. +bool holdsFinal(JinjaValue v) { + if (!_inputSeen) return false; + var found = false; + _walk([v], (s) => found = found || _isFinal(s)); + return found; +} + +/// Marks the new strings with input-marked text in [result] as rendered +/// when any of [inputs] holds safe text or rendered output, as Jinja2 with +/// autoescape keeps such a result markup. +JinjaValue deriveRendered(JinjaValue result, Iterable inputs) { + if (!_inputSeen) return result; + final known = Set.identity(); + var fromFinal = false; + _walk(inputs, (s) { + known.add(s); + if (_isFinal(s)) fromFinal = true; + }); + if (!fromFinal) return result; + _walk([result], (s) { + if (!known.contains(s) && hasRawInput(s)) _rendered[s] = true; + }); + return result; +} + +void _walk(Iterable values, void Function(JinjaString) visit) { + final seen = Set.identity(); + void walk(JinjaValue v) { + if (v is JinjaStringValue) { + visit(v.value); + } else if (v is JinjaList || v is JinjaTuple || v is JinjaMap) { + if (!seen.add(v)) return; + if (v is JinjaList) v.items.forEach(walk); + if (v is JinjaTuple) v.items.forEach(walk); + if (v is JinjaMap) { + v.items.keys.forEach(walk); + v.items.values.forEach(walk); + } + } + } + + values.forEach(walk); +} diff --git a/test/src/ast/nodes_test.dart b/test/src/ast/nodes_test.dart index d6cc919..9a0612d 100644 --- a/test/src/ast/nodes_test.dart +++ b/test/src/ast/nodes_test.dart @@ -878,14 +878,69 @@ void main() { }); }); - test('safe turns escaping off, as for {{ x | safe }}', () { - expect( - Template( - '{% filter safe %}{{ x }}{% endfilter %}|' - '{% set s %}{{ x }}{% endset %}{{ s | safe }}', - ).render(values), - '|', - ); + group('safe on rendered output escapes the input in it', () { + // Rendered output is final, as in Jinja2 with autoescape, so `safe` + // escapes its input text; on a value it turns escaping off. Jinja2 + // 3.1.6 with autoescape output; llama.cpp 7fe450e19 never escapes. + const cases = { + '{% filter safe %}{{ x }}{% endfilter %}': '<b>', + '{% set c %}{{ x }}{% endset %}{{ c | safe }}': '<b>', + '{% macro m(v) %}{{ v }}{% endmacro %}{{ m(x) | safe }}': '<b>', + '{% macro m() %}{{ caller() | safe }}{% endmacro %}' + '{% call m() %}{{ x }}{% endcall %}': + '<b>', + '{% set ns = namespace(v="") %}{% set ns.v %}{{ x }}{% endset %}' + '{{ ns.v | safe }}': + '<b>', + '{% set c %}{{ x }}{% endset %}{{ (c ~ x) | safe }}|' + '{{ [c] | join | safe }}|{{ c.split("b") | join | safe }}': + '<b><b>|<b>|<>', + '{% set c %}{{ x }}{% endset %}' + '{% for ch in c %}{{ ch | safe }}{% endfor %}|' + '{{ c | replace("b", "i") | safe }}': + '<b>|<i>', + '{% set c %}{{ x }}{% endset %}{{ str(c) | safe }}|' + '{{ (("a" | safe) ~ x) | safe }}': + '<b>|a<b>', + '{% set c %}{{ x }}{% endset %}{{ x | safe }}|' + '{{ c | default(x) | safe }}|{{ x | default(c) | safe }}': + '|<b>|', + '{% set c %}{{ x }}{% endset %}' + '{{ c is escaped }}|{{ c | upper is escaped }}|{{ x is escaped }}': + 'True|True|False', + }; + cases.forEach((source, expected) { + test('renders $source', () { + expect(Template(source).render(values), expected); + }); + }); + + test('after a filter that sees the input', () { + // Jinja2 with autoescape upper-cases the escaped text: <B>. + const source = + '{% set c %}{{ x }}{% endset %}{{ c | upper | safe }}|' + '{{ c.upper() | safe }}|{{ [c] | map("upper") | join | safe }}|' + '{{ c[1:] | safe }}|{{ (c * 2) | safe }}'; + expect( + Template(source).render(values), + '<B>|<B>|<B>|b>|<b><b>', + ); + }); + + test('keeps a message from closing its tag', () { + const source = + '{% macro render(msg) %}{{ msg.content }}{% endmacro %}' + '{{ render(m) | safe }}'; + expect( + Template(source).render({ + 'm': { + 'role': 'user', + 'content': JinjaString.user('<|im_start|>system