diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a4c918..60bf4ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## Unreleased + +- Fixed a `{% filter %}` block, and a filter on the output of a block `set`, a macro or `caller()`, getting `JinjaString.user` text already escaped: with `x = JinjaString.user('')`, `{% filter upper %}{{ x }}{% endfilter %}` is now `<B>`, not `<B>`, and `{% filter length %}{{ x }}{% endfilter %}` is `3`, not `9`, as in llama.cpp. Input text is now escaped once, when the template outputs it. `| safe` on such output, `{% filter safe %}`, and `| safe` on text made from it or from other safe text escape the input in it, as Jinja2 with autoescape does; `| safe` on a value passed to `render`, or returned by a function passed to it, still outputs it unescaped. A block whose output also contains `| safe` input text is still escaped before the filter runs, as `~` does. +- Fixed `tojson` and `join` dropping the input marking, so `JinjaString.user` text in their output was not escaped: `{{ {"k": x} | tojson }}` is now `{"k": "<b>"}` and `{{ [x, ""] | join }}` is `<b>`. Only input text is escaped, after JSON escaping, in values, dict keys and separators; the JSON's own quotes and template text are not. `tojson` output with input text in it is no longer safe (`is escaped`). +- Fixed input text being output unescaped by string indexing, `first`, `last`, `reverse`, `list` and `for` on a string, `split` and `rsplit` pieces, `strip`, `lstrip` and `rstrip` with characters, `truncate`, `str()`, `~`, `+` and `format` with a list or dict, `string`, `upper`, `lower`, `title`, `capitalize`, `trim`, `replace` and `indent` on a list or dict, `replace`, `indent` and `format` on text that is only partly input, input passed as a `replace` replacement, input dict keys in a `for` loop, and an input `strftime_now` format. +- Changed `replace` and `indent` on input text to leave the template text they add unescaped: `x | replace('b', '')` is now `<>`, not `<<i>>`, and `x | indent('> ', true)` no longer escapes `> `. Each character keeps the marking of the text it came from; llama.cpp marks all of the result as input when all of the source is. +- Fixed `| safe` text being escaped when passed to `format`, or to `replace` or `indent` on input text, sliced with a step or split by `unique`, and escaped twice when it held escaped input: `'{}'.format(x ~ (y | safe))` is now `<b>`, not `&lt;b&gt;`. +- Fixed member access on an undefined value giving a function, which printed as `` and was `defined`: for an undefined `y`, `{{ y.text }}` and `{{ y[0]['text'] }}` now print nothing and are not `defined`, as in llama.cpp. As in llama.cpp, `y.upper`, `y.length` and the other filters that llama.cpp gives undefined are functions returning an empty value (`y.length()` is `0`, `y.default(1)` is `1`) that print nothing, and calling any other member, such as `y.get()`, throws. Devstral Small 2507 no longer prints `` for an assistant turn without `content`. Also as in llama.cpp, Command R+ tool use renders an `object` parameter without `additionalProperties` as `Dict[str, Union[]]`, not `Dict[str, Any]`, and QwQ-32B throws for an assistant tool-call turn without `content` before the last turn. +- Changed a number subscript of none, as in `x[0]` or `x[0]['text']` for `x = none`, to throw `Cannot access property with non-string: got Integer`, as llama.cpp does; it gave undefined. Devstral Small 2507 with an assistant turn whose content is none now throws, as in llama.cpp. + ## 1.2.0 - Rewrote the README and example around rendering a chat template, and updated the pubspec description and topics. diff --git a/README.md b/README.md index 920dd82..51d1edb 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,8 @@ Real templates come from the `chat_template` field of a model's `tokenizer_confi Wrap untrusted values in `JinjaString.user`. They are escaped when rendered; a plain `String` is treated as template text and is not. +Input text keeps its marking through filters, `~`, loops, macros and `{% set %}` and `{% filter %}` blocks, and is escaped once, when the template outputs it. So a filter sees it as it was passed in: `{% filter length %}{{ name }}{% endfilter %}` counts the characters of `name`, not of its escaped form. `tojson` and `join` escape only the input text in their output, not the JSON's quotes or the template's separator. `| safe` and `markSafe()` turn escaping off for a value passed to `render`, or returned by a function passed to it. The output of a block `set`, macro, `caller()` or `{% filter %}` block is final, as in Jinja2 with autoescape, so `| safe` on it escapes the input in it. + ```dart import 'package:dinja/dinja.dart'; diff --git a/lib/src/ast/nodes.dart b/lib/src/ast/nodes.dart index 57deda7..fa455d2 100644 --- a/lib/src/ast/nodes.dart +++ b/lib/src/ast/nodes.dart @@ -1,5 +1,6 @@ import '../types/value.dart'; import '../types/jinja_string.dart'; +import '../types/marking.dart'; import '../types/repr.dart'; import 'dart:math' as math; import '../runtime/context.dart'; @@ -9,7 +10,7 @@ import '../lexer.dart'; // for Token /// Thrown by `break` and `continue`. Carries the output that the loop pass /// rendered before the statement, which the loop keeps. abstract class LoopSignal { - final List parts = []; + final List output = []; } class BreakSignal extends LoopSignal {} @@ -48,53 +49,56 @@ class Program extends Statement { @override String get type => 'Program'; + /// Renders the template. Input-marked text is escaped here, once. @override JinjaValue execute(Context ctx) { - return execStatements(body, ctx); + final result = renderScope( + ctx.environment.values, + () => execStatements(body, ctx), + ); + return JinjaStringValue((result as JinjaStringValue).value.escape()); } } -// ... helper to execute list of statements and gather output ... -/// Executes a list of [stmts] within the given [ctx] and aggregates their results. +/// Executes a list of [stmts] within the given [ctx] and joins their output. /// /// If a statement returns a string value, it is added to the output. /// Non-string values (except None and Undefined) are converted to strings. +/// Input-marked text stays unescaped, so a filter applied to the output sees +/// it as it was passed in; see [_joinOutput]. JinjaValue execStatements(List stmts, Context ctx) { - List parts = []; + final output = []; try { for (final stmt in stmts) { final val = stmt.execute(ctx); if (val is JinjaStringValue) { - if (val.isSafe) { - parts.addAll(val.value.parts); - } else { - // Check parts for input/unsafe content - for (final part in val.value.parts) { - if (part.isInput) { - final escaped = part.val - .replaceAll('&', '&') - .replaceAll('<', '<') - .replaceAll('>', '>') - .replaceAll('"', '"') - .replaceAll("'", '''); - // print('DEBUG escaping: ${part.val} -> $escaped'); - parts.add( - JinjaStringPart(escaped, part.isInput), - ); // Escaped -> safe/template - } else { - parts.add(part); - } - } - } + output.add(val.value); } else if (!val.isNone && !val.isUndefined) { - parts.addAll(reprOf(val).escape().parts); + output.add(reprOf(val)); } } } on LoopSignal catch (signal) { - signal.parts.insertAll(0, parts); + signal.output.insertAll(0, output); rethrow; } - return JinjaStringValue(JinjaString(parts, isSafe: true)); + return JinjaStringValue(_joinOutput(output)); +} + +/// Joins rendered [pieces]. +/// +/// Input-marked text stays unescaped unless a piece is a safe string with +/// input-marked text, whose text is final: then every piece is escaped and +/// the result is safe. +JinjaString _joinOutput(List pieces) { + final safeInput = pieces.any( + (p) => p.isSafe && p.parts.any((part) => part.isInput), + ); + if (pieces.any(hasRawInput) && !safeInput) { + return JinjaString([for (final p in pieces) ...p.parts]); + } + return JinjaString([ + for (final p in pieces) ...p.escape().parts, + ], isSafe: true); } /// Executes [stmts] whose output is captured rather than printed, as in a @@ -102,9 +106,11 @@ JinjaValue execStatements(List stmts, Context ctx) { /// `continue` leaving the body discards the captured output, as in llama.cpp. JinjaValue _execCaptured(List stmts, Context ctx) { try { - return execStatements(stmts, ctx); + final result = execStatements(stmts, ctx) as JinjaStringValue; + markRendered(result.value); + return result; } on LoopSignal catch (signal) { - signal.parts.clear(); + signal.output.clear(); rethrow; } } @@ -194,22 +200,23 @@ class ForStatement extends Statement { // Sort keys for deterministic output if needed, though Jinja2 doesn't strictly guarantee order, usually tests expect sorted keys or we should sort. // Standard Jinja2 iterates keys. // llama.cpp tests seem to expect keys. - final keys = iterableVal.asMap.keys.toList(); - // keys.sort(); // Optional: sort keys? Dinja existing tests expect 'a,b'. Maps in Dart preserve insertion order? LinkedHashMap does. - // If the map came from JSON/Literal, it's LinkedHashMap. - - items = keys.map((k) => JinjaStringValue.fromString(k)).toList(); + // Keys with the same string form are iterated once, as in asMap. + final keys = { + for (final k in iterableVal.items.keys) k.toString(): k, + }; + items = [ + for (final MapEntry(:key, :value) in keys.entries) + value is JinjaStringValue && hasRawInput(value.value) + ? value + : JinjaStringValue.fromString(key), + ]; } else if (iterableVal is JinjaList) { items = iterableVal.items; } else if (iterableVal is JinjaTuple) { items = iterableVal.items; } else if (iterableVal is JinjaStringValue) { - // Iterate chars - items = iterableVal.value.parts - .map((p) => p.val) - .expand((s) => s.split('')) - .map((c) => JinjaStringValue.fromString(c)) - .toList(); + items = charsOf(iterableVal.value); + deriveRendered(JinjaList(items), [iterableVal]); } // Filter items first if testExpr @@ -228,7 +235,7 @@ class ForStatement extends Statement { } // Loop - List parts = []; + final output = []; // As in Jinja2 and llama.cpp, the else block runs unless a pass finishes // without `break` or `continue`. var iterated = false; @@ -257,12 +264,12 @@ class ForStatement extends Statement { try { final result = execStatements(body, loopCtx); - if (result is JinjaStringValue) parts.addAll(result.value.parts); + if (result is JinjaStringValue) output.add(result.value); iterated = true; } on ContinueSignal catch (signal) { - parts.addAll(signal.parts); + output.addAll(signal.output); } on BreakSignal catch (signal) { - parts.addAll(signal.parts); + output.addAll(signal.output); break; } } @@ -270,15 +277,15 @@ class ForStatement extends Statement { if (!iterated) { try { final result = execStatements(defaultBlock, ctx); - if (result is JinjaStringValue) parts.addAll(result.value.parts); + if (result is JinjaStringValue) output.add(result.value); } on LoopSignal catch (signal) { // A `break` or `continue` in the else block belongs to an enclosing // loop and keeps the output of this one, as in Jinja2. - signal.parts.insertAll(0, parts); + signal.output.insertAll(0, output); rethrow; } } - return JinjaStringValue(JinjaString(parts, isSafe: true)); + return JinjaStringValue(_joinOutput(output)); } void _bindLoopVar(Context ctx, Expression loopVar, JinjaValue item) { @@ -714,9 +721,25 @@ class MemberExpression extends Expression { @override String get type => 'MemberExpression'; - @override @override JinjaValue execute(Context ctx) { + JinjaValue? obj; + final result = _access(ctx, (value) => obj = value); + final source = obj; + if (source == null) return result; + if (result is JinjaFunction) { + if (isSupplied(result) || !RenderInputs([source]).holdsFinal) { + return result; + } + return JinjaFunction(result.name, (args, kwargs) { + final inputs = RenderInputs([source, ...args, ...kwargs.values]); + return inputs.derive(result.handler(args, kwargs)); + }); + } + return deriveRendered(result, [source]); + } + + JinjaValue _access(Context ctx, void Function(JinjaValue) onObject) { // Handle slice expressions first, as they are a special case of computed access if (computed && property is SliceExpression) { final slice = property as SliceExpression; @@ -725,6 +748,7 @@ class MemberExpression extends Expression { final stepVal = slice.step?.execute(ctx); final obj = object.execute(ctx); + onObject(obj); int? start = (startVal != null && !startVal.isNone) ? startVal.asInt @@ -789,14 +813,15 @@ class MemberExpression extends Expression { return JinjaStringValue(s.substring(start, stop)); } + final raw = rawOf(s); final parts = []; if (step > 0) { for (int i = start; i < stop; i += step) { - if (i >= 0 && i < len) parts.addAll(s[i].parts); + if (i >= 0 && i < len) parts.addAll(raw[i].parts); } } else if (step < 0) { for (int i = start; i > stop; i += step) { - if (i >= 0 && i < len) parts.addAll(s[i].parts); + if (i >= 0 && i < len) parts.addAll(raw[i].parts); } } return JinjaStringValue(JinjaString(parts)); @@ -808,10 +833,19 @@ class MemberExpression extends Expression { if (computed || property is IntegerLiteral) { // Bracket notation: obj[expr], or an index after a dot: obj.0 final obj = object.execute(ctx); + onObject(obj); final prop = property.execute(ctx); if (!computed && prop.asInt < 0) { throw Exception('Static member property cannot be negative'); } + // As in llama.cpp, a subscript of undefined is undefined, and none + // takes only a string subscript. + if (obj.isUndefined || prop.isUndefined) return const JinjaUndefined(); + if (obj.isNone && prop is! JinjaStringValue) { + throw Exception( + 'Cannot access property with non-string: got ${prop.typeName}', + ); + } if (obj is JinjaMap) { // Bracket access: check keys FIRST @@ -851,7 +885,7 @@ class MemberExpression extends Expression { String s = obj.value.toString(); if (idx < 0) idx += s.length; if (idx >= 0 && idx < s.length) { - return JinjaStringValue.fromString(s[idx]); + return JinjaStringValue(rawOf(obj.value)[idx]); } } } @@ -869,6 +903,7 @@ class MemberExpression extends Expression { } final propName = (property as Identifier).name; final obj = object.execute(ctx); + onObject(obj); // Priority: 1. Attributes/Methods, 2. Map keys, 3. Special cases (length) final method = resolveMember(obj, propName); @@ -942,7 +977,10 @@ class CallExpression extends Expression { } } - return func.handler(positionals, kwargs); + return deriveRendered(called(func, func.handler(positionals, kwargs)), [ + ...positionals, + ...kwargs.values, + ]); } } @@ -1004,17 +1042,23 @@ class BinaryExpression extends Expression { } // String repeat? `~` is concat. `*` is repeat in Python. if (l is JinjaStringValue && r.isNumeric) { - return JinjaStringValue( - JinjaString([ - for (var i = 0; i < r.asInt; i++) ...l.value.parts, - ], isSafe: l.isSafe), + return deriveRendered( + JinjaStringValue( + JinjaString([ + for (var i = 0; i < r.asInt; i++) ...l.value.parts, + ], isSafe: l.isSafe), + ), + [l], ); } if (l is JinjaInteger && r is JinjaStringValue) { - return JinjaStringValue( - JinjaString([ - for (var i = 0; i < l.value; i++) ...r.value.parts, - ], isSafe: r.isSafe), + return deriveRendered( + JinjaStringValue( + JinjaString([ + for (var i = 0; i < l.value; i++) ...r.value.parts, + ], isSafe: r.isSafe), + ), + [r], ); } throw Exception('Invalid operand types for *'); @@ -1184,7 +1228,7 @@ class FilterExpression extends Expression { if (globalFilters.containsKey(filterName)) { final res = globalFilters[filterName]!(args, kwargs); - return res; + return deriveRendered(res, [...args, ...kwargs.values]); } throw Exception('Unknown filter: $filterName'); } @@ -1240,14 +1284,22 @@ class TestExpression extends Expression { } } -JinjaStringValue _concat(JinjaValue l, JinjaValue r) { - var left = l is JinjaStringValue ? l.value : JinjaString.template('$l'); - var right = r is JinjaStringValue ? r.value : JinjaString.template('$r'); - if (left.isSafe != right.isSafe) { - left = left.escape(); - right = right.escape(); - } - return JinjaStringValue(left + right); +JinjaValue _concat(JinjaValue l, JinjaValue r) => + deriveRendered(_join(l, r), [l, r]); + +JinjaStringValue _join(JinjaValue l, JinjaValue r) { + final left = _operand(l); + final right = _operand(r); + if (left.isSafe == right.isSafe) return JinjaStringValue(left + right); + final joined = _joinOutput([left, right]); + if (!joined.isSafe) return JinjaStringValue(joined); + return JinjaStringValue(left.escape() + right.escape()); +} + +JinjaString _operand(JinjaValue v) { + if (v is JinjaStringValue) return v.value; + final s = stringOf(v); + return hasRawInput(s) ? s : JinjaString.template('$v'); } /// Represents a selection expression used in filtering iterations. diff --git a/lib/src/runtime/builtins.dart b/lib/src/runtime/builtins.dart index aa9816d..15ef5b8 100644 --- a/lib/src/runtime/builtins.dart +++ b/lib/src/runtime/builtins.dart @@ -1,6 +1,7 @@ // ignore_for_file: non_constant_identifier_names import '../types/value.dart'; import '../types/jinja_string.dart'; +import '../types/marking.dart'; import '../types/repr.dart'; import 'strftime.dart'; import 'dart:convert'; @@ -64,26 +65,12 @@ final Map globalFilters = { 'format': _formatFilter, 'string': _string, 'title': (args, kwargs) => - (_resolveStringMember( - args.isNotEmpty && args[0] is JinjaStringValue - ? args[0] as JinjaStringValue - : JinjaStringValue.fromString( - args.isNotEmpty ? args[0].toString() : '', - ), - 'title', - ) - as JinjaFunction) - .handler(args, kwargs), + (_resolveStringMember(_asString(args), 'title') as JinjaFunction).handler( + args, + kwargs, + ), 'capitalize': (args, kwargs) => - (_resolveStringMember( - args.isNotEmpty && args[0] is JinjaStringValue - ? args[0] as JinjaStringValue - : JinjaStringValue.fromString( - args.isNotEmpty ? args[0].toString() : '', - ), - 'capitalize', - ) - as JinjaFunction) + (_resolveStringMember(_asString(args), 'capitalize') as JinjaFunction) .handler(args, kwargs), 'truncate': _truncate, 'wordcount': _wordcount, @@ -217,6 +204,18 @@ JinjaValue? resolveMember(JinjaValue obj, String name) { // Implementations +/// The first argument as a string value, keeping the input marking of the +/// strings in a list or dict. +JinjaStringValue _asString(List args) { + if (args.isEmpty) return JinjaStringValue.fromString(''); + final v = args[0]; + if (v is JinjaStringValue) return v; + final s = stringOf(v); + return hasRawInput(s) + ? JinjaStringValue(joinRaw([s])) + : JinjaStringValue.fromString(v.toString()); +} + JinjaValue _replaceFilter( List args, Map kwargs, @@ -224,9 +223,62 @@ JinjaValue _replaceFilter( if (args.isEmpty) return const JinjaStringValue(JinjaString([])); final obj = args[0]; if (args.length < 3) return obj; - final oldVal = args[1].toString(); - final newVal = args[2].toString(); - return _derived(obj, obj.toString().replaceAll(oldVal, newVal)); + final newVal = stringOf(args[2]); + return _derivedFrom( + obj, + _replaceIn(stringOf(obj), args[1].toString(), newVal, -1), + [newVal], + ); +} + +/// [s] with [old] replaced by [rep]: everywhere, as `String.replaceAll`, or +/// at most [count] times when it is not negative. Each character keeps the +/// input marking of the string it came from. +JinjaString _replaceIn(JinjaString s, String old, JinjaString rep, int count) { + s = rawOf(s); + rep = rawOf(rep); + if (count < 0) { + final text = s.toString(); + var prev = 0; + final parts = []; + for (final m in old.allMatches(text)) { + parts + ..addAll(s.substring(prev, m.start).parts) + ..addAll(rep.parts); + prev = m.end; + } + return JinjaString([...parts, ...s.substring(prev).parts]); + } + var res = s; + var start = 0; + for (var total = 0; total < count; total++) { + final idx = res.toString().indexOf(old, start); + if (idx == -1) break; + res = JinjaString([ + ...res.substring(0, idx).parts, + ...rep.parts, + ...res.substring(idx + old.length).parts, + ]); + start = idx + rep.length; + } + return res; +} + +/// [result], computed from [source] and the [inserted] strings. +/// +/// When any of them has input-marked text still to escape, each character +/// keeps the input marking of where it came from, so input text is escaped +/// and template text is not. Otherwise [result] is marked as `_derived` +/// marks it. +JinjaStringValue _derivedFrom( + JinjaValue source, + JinjaString result, [ + List inserted = const [], +]) { + if (hasRawInput(stringOf(source)) || inserted.any(hasRawInput)) { + return JinjaStringValue(joinRaw([result])); + } + return _derived(source, result.toString()); } JinjaStringValue _derived(JinjaValue source, String text) { @@ -239,8 +291,19 @@ JinjaStringValue _derived(JinjaValue source, String text) { ); } -JinjaStringValue _retext(JinjaStringValue source, String text) { - if (text.length != source.value.length) return _derived(source, text); +/// [text], a case change of [source], with the input marking of [source]. +/// +/// When the change alters the length and [source] has input-marked text to +/// escape, [perPart] computes it part by part instead. +JinjaStringValue _retext( + JinjaStringValue source, + String text, + JinjaString Function() perPart, +) { + if (text.length != source.value.length) { + if (hasRawInput(source.value)) return JinjaStringValue(perPart()); + return _derived(source, text); + } final parts = []; var start = 0; for (final part in source.value.parts) { @@ -293,31 +356,59 @@ JinjaValue _tojson(List args, Map kwargs) { final separators = separatorsArg is JinjaList || separatorsArg is JinjaTuple ? separatorsArg!.asList : const []; - final out = StringBuffer(); + final out = _JsonOut(); _writeJson( out, args[0], 0, indent: indent, itemSep: separators.isNotEmpty - ? separators[0].toString() - : (indent < 0 ? ', ' : ','), - keySep: separators.length > 1 ? separators[1].toString() : ': ', + ? stringOf(separators[0]) + : JinjaString.template(indent < 0 ? ', ' : ','), + keySep: separators.length > 1 + ? stringOf(separators[1]) + : JinjaString.template(': '), ensureAscii: arg('ensure_ascii', 1)?.asBool ?? false, sortKeys: arg('sort_keys', 4)?.asBool ?? false, ); - return JinjaStringValue( - JinjaString([JinjaStringPart(out.toString(), false)], isSafe: true), - ); + final json = joinRaw(out.pieces); + // Without input-marked text the output is safe template text, as before. + if (!json.parts.any((p) => p.isInput)) { + return JinjaStringValue( + JinjaString([JinjaStringPart(json.toString(), false)], isSafe: true), + ); + } + return JinjaStringValue(json); +} + +/// JSON output that keeps the input marking of the strings written to it. +class _JsonOut { + final List pieces = []; + + void write(Object text) => pieces.add(JinjaString.template('$text')); + + void writeString(JinjaString s, bool ensureAscii) { + write('"'); + pieces.add( + JinjaString([ + for (final p in rawOf(s).parts) + JinjaStringPart( + jsonEscape(p.val, ensureAscii: ensureAscii), + p.isInput, + ), + ]), + ); + write('"'); + } } void _writeJson( - StringBuffer out, + _JsonOut out, JinjaValue v, int level, { required int indent, - required String itemSep, - required String keySep, + required JinjaString itemSep, + required JinjaString keySep, required bool ensureAscii, required bool sortKeys, }) { @@ -335,7 +426,7 @@ void _writeJson( for (var i = 0; i < items.length; i++) { out.write(pad * (level + 1)); writeItem(items[i]); - if (i < items.length - 1) out.write(itemSep); + if (i < items.length - 1) out.pieces.add(itemSep); out.write(newline); } out.write(pad * level); @@ -361,16 +452,19 @@ void _writeJson( } else if (v is JinjaFloat) { out.write(_formatDouble(v.value)); } else if (v is JinjaStringValue) { - out.write('"${jsonEscape(v.toString(), ensureAscii: ensureAscii)}"'); + out.writeString(v.value, ensureAscii); } else if (v is JinjaList || v is JinjaTuple) { writeItems('[', ']', v.asList, writeValue); } else if (v is JinjaMap) { final entries = v.items.entries - .map((e) => MapEntry(e.key.toString(), e.value)) + .map((e) => MapEntry(stringOf(e.key), e.value)) .toList(); - if (sortKeys) entries.sort((a, b) => a.key.compareTo(b.key)); - writeItems('{', '}', entries, (MapEntry e) { - out.write('"${jsonEscape(e.key, ensureAscii: ensureAscii)}"$keySep'); + if (sortKeys) { + entries.sort((a, b) => a.key.toString().compareTo(b.key.toString())); + } + writeItems('{', '}', entries, (MapEntry e) { + out.writeString(e.key, ensureAscii); + out.pieces.add(keySep); writeValue(e.value); }); } else { @@ -462,15 +556,7 @@ JinjaValue _list(List args, Map kwargs) { final arg = args[0]; if (arg is JinjaList) return arg; if (arg is JinjaTuple) return JinjaList(List.from(arg.items)); - if (arg is JinjaStringValue) { - return JinjaList( - arg.value.parts - .map((p) => p.val) - .expand((s) => s.split('')) - .map((c) => JinjaStringValue.fromString(c)) - .toList(), - ); - } + if (arg is JinjaStringValue) return JinjaList(charsOf(arg.value)); if (arg is JinjaMap) { // list(dict) -> keys return JinjaList( @@ -519,7 +605,7 @@ JinjaValue _float(List args, Map kwargs) { JinjaValue _str(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); - return JinjaStringValue.fromString(args[0].toString()); + return JinjaStringValue(joinRaw([stringOf(args[0])])); } JinjaValue _len(List args, Map kwargs) { @@ -537,7 +623,7 @@ JinjaValue _first(List args, Map kwargs) { final v = args[0]; if (v is JinjaList && v.items.isNotEmpty) return v.items.first; if (v is JinjaStringValue && v.value.length > 0) { - return JinjaStringValue.fromString(v.value.toString()[0]); + return JinjaStringValue(rawOf(v.value)[0]); } if (v is JinjaTuple && v.items.isNotEmpty) return v.items.first; return const JinjaUndefined(); @@ -548,9 +634,7 @@ JinjaValue _last(List args, Map kwargs) { final v = args[0]; if (v is JinjaList && v.items.isNotEmpty) return v.items.last; if (v is JinjaStringValue && v.value.length > 0) { - return JinjaStringValue.fromString( - v.value.toString().substring(v.value.length - 1), - ); + return JinjaStringValue(rawOf(v.value)[v.value.length - 1]); } if (v is JinjaTuple && v.items.isNotEmpty) return v.items.last; return const JinjaUndefined(); @@ -739,11 +823,7 @@ JinjaValue _unique(List args, Map kwargs) { if (collection is JinjaList || collection is JinjaTuple) { items = collection.asList; } else if (collection is JinjaStringValue) { - items = [ - for (final part in collection.value.parts) - for (final char in part.val.split('')) - JinjaStringValue(JinjaString([JinjaStringPart(char, part.isInput)])), - ]; + items = charsOf(collection.value); } else if (collection is JinjaMap) { items = collection.items.keys.toList(); } else if (collection.isNone) { @@ -783,8 +863,8 @@ JinjaValue _reverse(List args, Map kwargs) { final v = args[0]; if (v is JinjaList) return JinjaList(v.items.reversed.toList()); if (v is JinjaStringValue) { - return JinjaStringValue.fromString( - v.value.toString().split('').reversed.join(''), + return JinjaStringValue( + joinRaw(charsOf(v.value).reversed.map((c) => stringOf(c))), ); } return v; @@ -976,11 +1056,13 @@ JinjaValue _join(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); final collection = args[0]; if (collection is! JinjaList && collection is! JinjaTuple) { - return JinjaStringValue.fromString(collection.toString()); + return JinjaStringValue(joinRaw([stringOf(collection)])); } - final delimiter = - (kwargs['d'] ?? (args.length > 1 ? args[1] : null))?.toString() ?? ''; + final delimiterArg = kwargs['d'] ?? (args.length > 1 ? args[1] : null); + final delimiter = delimiterArg == null + ? JinjaString.template('') + : stringOf(delimiterArg); final attribute = kwargs['attribute']?.toString() ?? (args.length > 2 ? args[2].toString() : null); @@ -989,14 +1071,16 @@ JinjaValue _join(List args, Map kwargs) { ? collection.items : (collection as JinjaTuple).items; - final strings = items - .map((e) { - final val = attribute != null ? _resolveAttribute(e, attribute) : e; - return val.toString(); - }) - .join(delimiter); - - return JinjaStringValue.fromString(strings); + return JinjaStringValue( + joinRaw([ + for (var i = 0; i < items.length; i++) ...[ + if (i > 0) delimiter, + stringOf( + attribute != null ? _resolveAttribute(items[i], attribute) : items[i], + ), + ], + ]), + ); } JinjaValue _safe(List args, Map kwargs) { @@ -1005,6 +1089,7 @@ JinjaValue _safe(List args, Map kwargs) { } final v = args[0]; if (v is JinjaStringValue) { + if (isRendered(v.value)) return JinjaStringValue(v.value.escape()); return JinjaStringValue(v.value.markSafe()); } return JinjaStringValue(JinjaString.from(v.toString(), isSafe: true)); @@ -1280,7 +1365,8 @@ JinjaValue _testIsEscaped( Map kwargs, ) { if (args.isEmpty) return const JinjaBoolean(false); - return JinjaBoolean(args[0].isSafe); + final v = args[0]; + return JinjaBoolean(v.isSafe || v is JinjaStringValue && isRendered(v.value)); } JinjaValue _testIsFilter( @@ -1455,13 +1541,14 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { final maxsplit = kwargs['maxsplit']?.asInt ?? (args.length > 1 ? args[1].asInt : -1); - String s = obj.value.toString(); - List parts; + final src = rawOf(obj.value); + final s = src.toString(); + List parts; if (delimiter == null || delimiter == ' ') { // split by whitespace - parts = s.trim().split(RegExp(r'\s+')); + parts = _splitWhitespace(src); if (maxsplit >= 0 && parts.length > maxsplit + 1) { - final rest = parts.sublist(maxsplit).join(' '); + final rest = _joinSpaces(parts.sublist(maxsplit)); parts = parts.sublist(0, maxsplit)..add(rest); } } else { @@ -1472,20 +1559,18 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { for (int i = 0; i < maxsplit; i++) { int idx = s.indexOf(delimiter, start); if (idx == -1) break; - parts.add(s.substring(start, idx)); + parts.add(src.substring(start, idx)); start = idx + delimiter.length; } - parts.add(s.substring(start)); + parts.add(src.substring(start)); } else { - parts = s.split(delimiter); + parts = _splitOn(src, delimiter); } } return JinjaList([ for (var i = 0; i < parts.length; i++) - i == parts.length - 1 - ? _derived(obj, parts[i]) - : JinjaStringValue.fromString(parts[i]), + _splitPiece(obj, parts[i], last: i == parts.length - 1), ]); }); case 'rsplit': @@ -1494,13 +1579,14 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { final maxsplit = kwargs['maxsplit']?.asInt ?? (args.length > 1 ? args[1].asInt : -1); - String s = obj.value.toString(); - List parts; + final src = rawOf(obj.value); + final s = src.toString(); + List parts; if (delimiter == null || delimiter == ' ') { // rsplit by whitespace - parts = s.trim().split(RegExp(r'\s+')); + parts = _splitWhitespace(src); if (maxsplit >= 0 && parts.length > maxsplit + 1) { - final rest = parts.sublist(0, parts.length - maxsplit).join(' '); + final rest = _joinSpaces(parts.sublist(0, parts.length - maxsplit)); parts = [rest, ...parts.sublist(parts.length - maxsplit)]; } } else { @@ -1510,26 +1596,28 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { for (int i = 0; i < maxsplit; i++) { int idx = s.lastIndexOf(delimiter, end - 1); if (idx == -1) break; - parts.insert(0, s.substring(idx + delimiter.length, end)); + parts.insert(0, src.substring(idx + delimiter.length, end)); end = idx; } - parts.insert(0, s.substring(0, end)); + parts.insert(0, src.substring(0, end)); } else { - parts = s.split(delimiter); + parts = _splitOn(src, delimiter); } } return JinjaList([ for (var i = 0; i < parts.length; i++) - i == 0 - ? _derived(obj, parts[i]) - : JinjaStringValue.fromString(parts[i]), + _splitPiece(obj, parts[i], last: i == 0), ]); }); case 'capitalize': return JinjaFunction('capitalize', (args, kwargs) { final s = obj.value.toString(); if (s.isEmpty) return obj; - return _retext(obj, s[0].toUpperCase() + s.substring(1).toLowerCase()); + return _retext( + obj, + s[0].toUpperCase() + s.substring(1).toLowerCase(), + () => _capitalized(obj.value), + ); }); case 'title': return JinjaFunction('title', (args, kwargs) { @@ -1543,6 +1631,18 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { return w[0].toUpperCase() + w.substring(1).toLowerCase(); }) .join(' '), + () { + final words = []; + var start = 0; + for (final w in s.split(' ')) { + if (start > 0) words.add(obj.value.substring(start - 1, start)); + words.add( + _capitalized(obj.value.substring(start, start + w.length)), + ); + start += w.length + 1; + } + return joinRaw(words); + }, ); }); case 'format': @@ -1550,67 +1650,113 @@ JinjaValue? _resolveStringMember(JinjaStringValue obj, String name) { case 'replace': return JinjaFunction('replace', (args, kwargs) { if (args.length < 2) return obj; - final oldVal = args[0].toString(); - final newVal = args[1].toString(); + final newVal = stringOf(args[1]); final count = kwargs['count']?.asInt ?? (args.length > 2 ? args[2].asInt : -1); - - String s = obj.value.toString(); - if (count >= 0) { - String res = s; - int total = 0; - int start = 0; - while (total < count) { - int idx = res.indexOf(oldVal, start); - if (idx == -1) break; - res = res.replaceRange(idx, idx + oldVal.length, newVal); - start = idx + newVal.length; - total++; - } - return _derived(obj, res); - } - - return _derived(obj, s.replaceAll(oldVal, newVal)); + return _derivedFrom( + obj, + _replaceIn(obj.value, args[0].toString(), newVal, count), + [newVal], + ); }); } return null; } +/// `s.trim().split(RegExp(r'\s+'))`, keeping input marking. +List _splitWhitespace(JinjaString s) { + final text = s.toString(); + final trimmed = text.trim(); + final lead = text.length - text.trimLeft().length; + final src = s.substring(lead, lead + trimmed.length); + final pieces = []; + var prev = 0; + for (final m in RegExp(r'\s+').allMatches(trimmed)) { + pieces.add(src.substring(prev, m.start)); + prev = m.end; + } + return pieces..add(src.substring(prev)); +} + +/// `s.split(delimiter)`, keeping input marking. +List _splitOn(JinjaString s, String delimiter) { + final text = s.toString(); + if (delimiter.isEmpty) { + return [for (var i = 0; i < text.length; i++) s.substring(i, i + 1)]; + } + final pieces = []; + var prev = 0; + for (final m in delimiter.allMatches(text)) { + pieces.add(s.substring(prev, m.start)); + prev = m.end; + } + return pieces..add(s.substring(prev)); +} + +/// [pieces] joined with single spaces. +JinjaString _joinSpaces(List pieces) => JinjaString([ + for (var i = 0; i < pieces.length; i++) ...[ + if (i > 0) const JinjaStringPart(' ', false), + ...pieces[i].parts, + ], +]); + +/// A piece of a `split` or `rsplit` of [source]. +/// +/// With input-marked text still to escape, the piece keeps its marking. +/// Otherwise only the [last] piece split off, the remainder, is marked, as +/// in llama.cpp: as input when all of [source] is. +JinjaStringValue _splitPiece( + JinjaStringValue source, + JinjaString piece, { + required bool last, +}) { + if (hasRawInput(source.value)) return JinjaStringValue(joinRaw([piece])); + if (last) return _derived(source, piece.toString()); + return JinjaStringValue.fromString(piece.toString()); +} + +/// [s] with its first character upper case and the rest lower case, changed +/// part by part. +JinjaString _capitalized(JinjaString s) => + joinRaw([s.substring(0, 1).toUpperCase(), s.substring(1).toLowerCase()]); + +/// Formats [args] into the `{}` placeholders of [fmt]. The literal text and +/// each argument keep their input marking. JinjaStringValue _format(JinjaStringValue fmt, List args) { - final literalIsInput = fmt.value.allPartsAreInput; - final source = fmt.value.toString(); + final format = rawOf(fmt.value); + final source = format.toString(); final parts = []; - final literal = StringBuffer(); - void flushLiteral() { - if (literal.isEmpty) return; - parts.add(JinjaStringPart(literal.toString(), literalIsInput)); - literal.clear(); + var literalStart = 0; + void flushLiteral(int end) { + parts.addAll(format.substring(literalStart, end).parts); } var next = 0; for (var i = 0; i < source.length; i++) { - if (source[i] != '{') { - literal.write(source[i]); - continue; - } + if (source[i] != '{') continue; if (i + 1 >= source.length || source[i + 1] != '}') { throw Exception("format() only supports simple '{}' placeholders"); } - i++; if (next >= args.length) { throw Exception( 'format() expected at least ${next + 1} arguments, got ${args.length}', ); } - flushLiteral(); + flushLiteral(i); + i++; + literalStart = i + 1; final arg = args[next++]; + final text = stringOf(arg); parts.addAll( arg is JinjaStringValue - ? arg.value.parts + ? rawOf(text).parts + : hasRawInput(text) + ? joinRaw([text]).parts : [JinjaStringPart('$arg', false)], ); } - flushLiteral(); + flushLiteral(source.length); return JinjaStringValue(JinjaString(parts)); } @@ -1640,6 +1786,9 @@ JinjaValue _strip(List args, Map kwargs) { if (chars == null) { if (v is JinjaStringValue) return JinjaStringValue(v.value.trim()); + if (hasRawInput(stringOf(v))) { + return JinjaStringValue(_asString(args).value.trim()); + } return JinjaStringValue.fromString(v.toString().trim()); } @@ -1654,9 +1803,14 @@ JinjaValue _strip(List args, Map kwargs) { while (end > start && charSet.contains(s[end - 1])) { end--; } - return JinjaStringValue.fromString(s.substring(start, end)); + return _codeUnits(v, start, end); } +/// Code units [start] to [end] of [v]'s string form, keeping their input +/// marking. +JinjaStringValue _codeUnits(JinjaValue v, int start, [int? end]) => + JinjaStringValue(joinRaw([stringOf(v).substring(start, end)])); + JinjaValue _lstrip(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); final v = args[0]; @@ -1666,6 +1820,9 @@ JinjaValue _lstrip(List args, Map kwargs) { if (chars == null) { if (v is JinjaStringValue) return JinjaStringValue(v.value.trimLeft()); + if (hasRawInput(stringOf(v))) { + return JinjaStringValue(_asString(args).value.trimLeft()); + } return JinjaStringValue.fromString(v.toString().trimLeft()); } @@ -1675,7 +1832,7 @@ JinjaValue _lstrip(List args, Map kwargs) { while (start < s.length && charSet.contains(s[start])) { start++; } - return JinjaStringValue.fromString(s.substring(start)); + return _codeUnits(v, start); } JinjaValue _rstrip(List args, Map kwargs) { @@ -1687,6 +1844,9 @@ JinjaValue _rstrip(List args, Map kwargs) { if (chars == null) { if (v is JinjaStringValue) return JinjaStringValue(v.value.trimRight()); + if (hasRawInput(stringOf(v))) { + return JinjaStringValue(_asString(args).value.trimRight()); + } return JinjaStringValue.fromString(v.toString().trimRight()); } @@ -1696,7 +1856,7 @@ JinjaValue _rstrip(List args, Map kwargs) { while (end > 0 && charSet.contains(s[end - 1])) { end--; } - return JinjaStringValue.fromString(s.substring(0, end)); + return _codeUnits(v, 0, end); } JinjaValue? _resolveNoneMember(JinjaNone obj, String name) { @@ -1716,9 +1876,63 @@ JinjaValue? _resolveNoneMember(JinjaNone obj, String name) { return null; } +/// The members of undefined in llama.cpp, which return an empty value of +/// the filter's result type. +final Map _undefinedMembers = { + for (final name in [ + 'capitalize', + 'join', + 'lower', + 'replace', + 'safe', + 'string', + 'strip', + 'title', + 'truncate', + 'upper', + ]) + name: const JinjaStringValue(JinjaString([])), + for (final name in [ + 'items', + 'list', + 'map', + 'reject', + 'rejectattr', + 'reverse', + 'select', + 'selectattr', + 'sort', + 'unique', + ]) + name: const JinjaList([]), + for (final name in ['length', 'sum', 'wordcount']) + name: const JinjaInteger(0), + for (final name in ['first', 'last', 'max', 'min']) + name: const JinjaUndefined(), +}; + JinjaValue? _resolveUndefinedMember(JinjaUndefined obj, String name) { - // undefined methods mostly return undefined or empty - return JinjaFunction(name, (args, kwargs) => const JinjaUndefined()); + if (name == 'default') { + return _UndefinedMember( + name, + (args, kwargs) => _default([obj, ...args], kwargs), + ); + } + final empty = _undefinedMembers[name]; + if (empty == null) return null; + return _UndefinedMember(name, (args, kwargs) { + // A new list, as a template may append to it. + return empty is JinjaList ? JinjaList([]) : empty; + }); +} + +/// A member of undefined, such as `x.upper` for an undefined `x`. As in +/// llama.cpp, it prints as nothing. +class _UndefinedMember extends JinjaFunction { + const _UndefinedMember(super.name, super.handler); + + @override + String toString() => ''; } JinjaValue _namespace(List args, Map kwargs) { @@ -1799,8 +2013,8 @@ JinjaValue _dictsort(List args, Map kwargs) { JinjaValue _upper(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); final v = args[0]; - if (v is JinjaStringValue) { - return JinjaStringValue(v.value.toUpperCase()); + if (v is JinjaStringValue || hasRawInput(stringOf(v))) { + return JinjaStringValue(_asString(args).value.toUpperCase()); } return JinjaStringValue.fromString(v.toString().toUpperCase()); } @@ -1808,15 +2022,16 @@ JinjaValue _upper(List args, Map kwargs) { JinjaValue _lower(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); final v = args[0]; - if (v is JinjaStringValue) { - return JinjaStringValue(v.value.toLowerCase()); + if (v is JinjaStringValue || hasRawInput(stringOf(v))) { + return JinjaStringValue(_asString(args).value.toLowerCase()); } return JinjaStringValue.fromString(v.toString().toLowerCase()); } JinjaValue _indent(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); - final str = args[0].toString(); + final source = rawOf(stringOf(args[0])); + final str = source.toString(); final width = args.length > 1 ? args[1] : kwargs['width']; final first = args.length > 2 ? args[2].asBool @@ -1826,30 +2041,44 @@ JinjaValue _indent(List args, Map kwargs) { : (kwargs['blank']?.asBool ?? false); final indentStr = width is JinjaStringValue - ? width.toString() - : ' ' * (width?.asInt ?? 4); - final lines = str.isEmpty ? [] : str.split('\n'); + ? width.value + : JinjaString.template(' ' * (width?.asInt ?? 4)); + // Each line as a slice of [source], and the newline after it. + final lines = []; + final newlines = []; + if (str.isNotEmpty) { + var start = 0; + for (final m in '\n'.allMatches(str)) { + lines.add(source.substring(start, m.start)); + newlines.add(source.substring(m.start, m.end)); + start = m.end; + } + lines.add(source.substring(start)); + } final trailingNewline = str.endsWith('\n'); if (trailingNewline) lines.removeLast(); - final buffer = StringBuffer(); + final out = []; for (var i = 0; i < lines.length; i++) { final line = lines[i]; - if (i > 0) buffer.write('\n'); - if (i == 0 ? first : (line.isNotEmpty || blank)) buffer.write(indentStr); - buffer.write(line); + if (i > 0) out.add(newlines[i - 1]); + if (i == 0 ? first : (line.length > 0 || blank)) out.add(indentStr); + out.add(line); } if (trailingNewline) { - buffer.write('\n'); - if (blank) buffer.write(indentStr); + out.add(newlines.last); + if (blank) out.add(indentStr); } - return _derived(args[0], buffer.toString()); + return _derivedFrom( + args[0], + JinjaString([for (final piece in out) ...rawOf(piece).parts]), + [indentStr], + ); } JinjaValue _string(List args, Map kwargs) { if (args.isEmpty) return const JinjaStringValue(JinjaString([])); - if (args[0] is JinjaStringValue) return args[0]; - return JinjaStringValue.fromString(args[0].toString()); + return _asString(args); } JinjaValue _strftime_now( @@ -1868,6 +2097,10 @@ JinjaValue _strftime_now( if (result.isEmpty || utf8.encode(result).length >= 100) { throw Exception('strftime_now: failed to format time'); } + // Text of an input-marked format may pass through, so it is escaped. + if (hasRawInput(format.value)) { + return JinjaStringValue(JinjaString.user(result)); + } return JinjaStringValue(JinjaString.from(result, isSafe: true)); } @@ -1890,25 +2123,27 @@ JinjaValue _truncate(List args, Map kwargs) { kwargs['length']?.asInt ?? (args.length > 1 ? args[1].asInt : 255); final killwords = kwargs['killwords']?.asBool ?? (args.length > 2 ? args[2].asBool : false); - final end = - kwargs['end']?.toString() ?? - (args.length > 3 ? args[3].toString() : '...'); + final endArg = kwargs['end'] ?? (args.length > 3 ? args[3] : null); + final end = endArg == null ? JinjaString.template('...') : stringOf(endArg); if (s.length <= length) return args[0]; - String res; + int keep; if (killwords) { - res = s.substring(0, length - end.length); + keep = length - end.length; } else { // find last whitespace before length int lastSpace = s.lastIndexOf(' ', length - end.length); if (lastSpace == -1) { - res = s.substring(0, length - end.length); + keep = length - end.length; } else { - res = s.substring(0, lastSpace); + keep = lastSpace; } } - return JinjaStringValue.fromString(res + end); + final kept = s.substring(0, keep); + return JinjaStringValue( + joinRaw([stringOf(args[0]).substring(0, kept.length), end]), + ); } JinjaValue _wordcount(List args, Map kwargs) { diff --git a/lib/src/types/marking.dart b/lib/src/types/marking.dart new file mode 100644 index 0000000..517fd20 --- /dev/null +++ b/lib/src/types/marking.dart @@ -0,0 +1,168 @@ +import 'jinja_string.dart'; +import 'repr.dart'; +import 'value.dart'; + +/// Whether [s] has input-marked text that is escaped on output. +bool hasRawInput(JinjaString s) => !s.isSafe && s.parts.any((p) => p.isInput); + +/// [s] as unescaped text. The text of a safe string is final, so its +/// input-marked parts become template text. +JinjaString rawOf(JinjaString s) { + if (!s.isSafe) return s; + return JinjaString([ + for (final p in s.parts) p.isInput ? JinjaStringPart(p.val, false) : p, + ]); +} + +/// The string form of [v] that keeps the input marking of the strings in it: +/// a string itself, a list, tuple or dict as [reprOf], anything else as +/// template text. +JinjaString stringOf(JinjaValue v) { + if (v is JinjaStringValue) return v.value; + if (v is JinjaList || v is JinjaTuple || v is JinjaMap) return reprOf(v); + return JinjaString.template(v.toString()); +} + +/// Joins [pieces] as unescaped text. +/// +/// Without input-marked text the result is one template part, as a plain +/// string is; otherwise each character keeps the marking of its piece. +JinjaString joinRaw(Iterable pieces) { + final parts = []; + for (final part in [for (final piece in pieces) ...rawOf(piece).parts]) { + if (part.val.isEmpty) continue; + if (parts.isNotEmpty && parts.last.isInput == part.isInput) { + parts.last = JinjaStringPart(parts.last.val + part.val, part.isInput); + } else { + parts.add(part); + } + } + if (!parts.any((p) => p.isInput)) { + return JinjaString.template(parts.map((p) => p.val).join()); + } + return JinjaString(parts); +} + +/// The characters of [s], each keeping its input marking. +List charsOf(JinjaString s) => [ + for (final p in rawOf(s).parts) + for (final c in p.val.split('')) + JinjaStringValue(JinjaString([JinjaStringPart(c, p.isInput)])), +]; + +/// The rendered output of one render and the values its caller supplied. +class _Render { + final Set rendered = Set.identity(); + final Set supplied = Set.identity(); + final Set functions = Set.identity(); + + /// Whether the caller supplied input-marked text or a function, the only + /// sources of input-marked text in a render. + bool get tracking => supplied.isNotEmpty || functions.isNotEmpty; + + void supply(Iterable values) => _walk(values, (s) { + if (hasRawInput(s)) supplied.add(s); + }, onFunction: functions.add); +} + +_Render? _render; + +/// Runs [render] over the caller's [values], with its own record of +/// rendered output, restored afterwards. +T renderScope(Iterable values, T Function() render) { + final saved = _render; + _render = _Render()..supply(values); + try { + return render(); + } finally { + _render = saved; + } +} + +/// Records [s] as rendered output: a block `set`, macro, `caller()` or +/// `filter` block body. Its text is final, as in Jinja2 with autoescape, so +/// `safe` escapes the input-marked text in it. +void markRendered(JinjaString s) => _render?.rendered.add(s); + +/// Whether [s] is rendered output, or derived from it or from safe text, in +/// the current render. +bool isRendered(JinjaString s) => _render?.rendered.contains(s) ?? false; + +/// Whether [f] was supplied by the caller of the current render. +bool isSupplied(JinjaFunction f) => _render?.functions.contains(f) ?? false; + +/// Records the [result] of a call to [f]: a value from a function the caller +/// supplied is the caller's, and is never marked rendered. +JinjaValue called(JinjaFunction f, JinjaValue result) { + if (isSupplied(f)) _render!.supply([result]); + return result; +} + +/// The strings in some values, and whether any is safe text or rendered +/// output, taken before an operation on them. +class RenderInputs { + final Set _known = Set.identity(); + var _final = false; + + RenderInputs(Iterable values) { + final render = _render; + if (render == null || !render.tracking) return; + _walk(values, (s) { + _known.add(s); + if (s.isSafe || render.rendered.contains(s)) _final = true; + }); + } + + /// Whether the values hold safe text or rendered output. + bool get holdsFinal => _final; + + /// Marks the new strings with input-marked text in [result] as rendered + /// when these values hold safe text or rendered output, as Jinja2 with + /// autoescape keeps such a result markup. + JinjaValue derive(JinjaValue result) { + final render = _render; + if (render == null || !_final) return result; + _walk([result], (s) { + if (hasRawInput(s) && !_known.contains(s)) render.rendered.add(s); + }); + return result; + } +} + +/// [RenderInputs.derive] for an operation that does not change [inputs], +/// leaving the values the caller supplied unmarked. +JinjaValue deriveRendered(JinjaValue result, Iterable inputs) { + final render = _render; + if (render == null || !render.tracking) return result; + var fresh = false; + _walk([result], (s) { + if (hasRawInput(s) && !render.supplied.contains(s)) fresh = true; + }); + if (!fresh) return result; + return RenderInputs(inputs).derive(result); +} + +void _walk( + Iterable values, + void Function(JinjaString) visit, { + void Function(JinjaFunction)? onFunction, +}) { + final seen = Set.identity(); + void walk(JinjaValue v) { + if (v is JinjaStringValue) { + visit(v.value); + } else if (v is JinjaFunction) { + onFunction?.call(v); + } else if (v is JinjaList || v is JinjaTuple || v is JinjaMap) { + if (!seen.add(v)) return; + if (v is JinjaList) v.items.forEach(walk); + if (v is JinjaTuple) v.items.forEach(walk); + if (v is JinjaMap) { + v.items.keys.forEach(walk); + v.items.values.forEach(walk); + } + } + } + + values.forEach(walk); +} diff --git a/test/src/ast/nodes_test.dart b/test/src/ast/nodes_test.dart index f698ade..9a0612d 100644 --- a/test/src/ast/nodes_test.dart +++ b/test/src/ast/nodes_test.dart @@ -833,4 +833,241 @@ void main() { }); } }); + group('Filters on captured output see the input', () { + // A filter block, or a filter on the output of a block set, macro or + // caller(), sees input-marked text as it was passed in, as `x | upper` + // does. Escaping happens once, at output. llama.cpp 7fe450e19 and + // Jinja2 3.1.6 without autoescape give each output with the input + // unescaped; Jinja2 with autoescape filters the escaped text instead. + final values = {'x': JinjaString.user('')}; + const cases = { + '{% filter upper %}{{ x }}{% endfilter %}': '<B>', + '{% set s %}{{ x }}{% endset %}{{ s | upper }}': '<B>', + '{% filter replace("b", "i") %}{{ x }}{% endfilter %}': '<i>', + '{% filter upper %}{% if true %}{{ x }}{% endif %}{% endfilter %}': + '<B>', + '{% macro m() %}{{ x }}{% endmacro %}{{ m() | upper }}': '<B>', + '{% set s %}{% for i in [1] %}{{ x }}{% endfor %}{% endset %}' + '{{ s | replace("<", "[") }}': + '[b>', + '{% filter trim %} {{ x }} {% endfilter %}': '<b>', + '{% filter upper %}{{ x }}{% endfilter %}': '<B>', + '{% set s %}{{ x }}{% endset %}{{ s }}|{{ s ~ x }}': + '<b>|<b><b>', + '{% set s %}a{% endset %}{% filter upper %}{{ s ~ x }}{% endfilter %}': + 'A<B>', + }; + cases.forEach((source, expected) { + test('renders $source', () { + expect(Template(source).render(values), expected); + }); + }); + + // llama.cpp 7fe450e19 output; Jinja2 raises for the filter block and + // counts the escaped text with autoescape. + const lengths = { + '{% filter length %}{{ x }}{% endfilter %}': '3', + '{% set s %}{{ x }}{% endset %}{{ s | length }}': '3', + '{% macro m() %}{{ caller() | length }}{% endmacro %}' + '{% call m() %}{{ x }}{% endcall %}': + '3', + }; + lengths.forEach((source, expected) { + test('renders $source', () { + expect(Template(source).render(values), expected); + }); + }); + + group('safe on rendered output escapes the input in it', () { + // Rendered output is final, as in Jinja2 with autoescape, so `safe` + // escapes its input text; on a value it turns escaping off. Jinja2 + // 3.1.6 with autoescape output; llama.cpp 7fe450e19 never escapes. + const cases = { + '{% filter safe %}{{ x }}{% endfilter %}': '<b>', + '{% set c %}{{ x }}{% endset %}{{ c | safe }}': '<b>', + '{% macro m(v) %}{{ v }}{% endmacro %}{{ m(x) | safe }}': '<b>', + '{% macro m() %}{{ caller() | safe }}{% endmacro %}' + '{% call m() %}{{ x }}{% endcall %}': + '<b>', + '{% set ns = namespace(v="") %}{% set ns.v %}{{ x }}{% endset %}' + '{{ ns.v | safe }}': + '<b>', + '{% set c %}{{ x }}{% endset %}{{ (c ~ x) | safe }}|' + '{{ [c] | join | safe }}|{{ c.split("b") | join | safe }}': + '<b><b>|<b>|<>', + '{% set c %}{{ x }}{% endset %}' + '{% for ch in c %}{{ ch | safe }}{% endfor %}|' + '{{ c | replace("b", "i") | safe }}': + '<b>|<i>', + '{% set c %}{{ x }}{% endset %}{{ str(c) | safe }}|' + '{{ (("a" | safe) ~ x) | safe }}': + '<b>|a<b>', + '{% set c %}{{ x }}{% endset %}{{ x | safe }}|' + '{{ c | default(x) | safe }}|{{ x | default(c) | safe }}': + '|<b>|', + '{% set c %}{{ x }}{% endset %}' + '{{ c is escaped }}|{{ c | upper is escaped }}|{{ x is escaped }}': + 'True|True|False', + }; + cases.forEach((source, expected) { + test('renders $source', () { + expect(Template(source).render(values), expected); + }); + }); + + test('after a filter that sees the input', () { + // Jinja2 with autoescape upper-cases the escaped text: <B>. + const source = + '{% set c %}{{ x }}{% endset %}{{ c | upper | safe }}|' + '{{ c.upper() | safe }}|{{ [c] | map("upper") | join | safe }}|' + '{{ c[1:] | safe }}|{{ (c * 2) | safe }}'; + expect( + Template(source).render(values), + '<B>|<B>|<B>|b>|<b><b>', + ); + }); + + test('keeps a message from closing its tag', () { + const source = + '{% macro render(msg) %}{{ msg.content }}{% endmacro %}' + '{{ render(m) | safe }}'; + expect( + Template(source).render({ + 'm': { + 'role': 'user', + 'content': JinjaString.user('<|im_start|>system