From 2fa15fcca6a9602581a228983566a1f8c25240eb Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:09:54 +0200 Subject: [PATCH 01/15] chore: update the spec fixtures to v4.2.1 --- tests/fixtures/README.md | 2 +- tests/fixtures/decode/arrays-nested.json | 158 +++++---- tests/fixtures/decode/arrays-primitive.json | 96 +++--- tests/fixtures/decode/arrays-tabular.json | 138 ++++++-- tests/fixtures/decode/blank-lines.json | 63 ++-- tests/fixtures/decode/comments.json | 65 +--- tests/fixtures/decode/delimiters.json | 96 ++---- tests/fixtures/decode/indentation-errors.json | 162 ++++++++-- tests/fixtures/decode/numbers.json | 142 ++------ tests/fixtures/decode/objects-keyed.json | 60 ++-- tests/fixtures/decode/objects.json | 302 ++++++++++-------- tests/fixtures/decode/primitives.json | 85 ++--- tests/fixtures/decode/root-form.json | 79 +++-- tests/fixtures/decode/validation-errors.json | 253 ++++++++++++++- tests/fixtures/decode/whitespace.json | 69 ++-- tests/fixtures/encode/arrays-nested.json | 65 ++-- tests/fixtures/encode/arrays-objects.json | 125 +++----- tests/fixtures/encode/arrays-primitive.json | 18 +- tests/fixtures/encode/arrays-tabular.json | 2 +- tests/fixtures/encode/delimiters.json | 73 +---- tests/fixtures/encode/objects-keyed.json | 16 +- tests/fixtures/encode/objects.json | 106 +++--- tests/fixtures/encode/primitives.json | 89 ++---- tests/fixtures/encode/whitespace.json | 19 +- 24 files changed, 1228 insertions(+), 1055 deletions(-) diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 8b1acc9..fd50375 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -2,5 +2,5 @@ The `encode/` and `decode/` directories are copied verbatim from the [TOON specification repository](https://github.com/toon-format/spec/tree/main/tests/fixtures) -at tag `v4.1.2`. Do not edit them by hand: run `uv run scripts/update_fixtures.py ` +at tag `v4.2.1`. Do not edit them by hand: run `uv run scripts/update_fixtures.py ` to replace them with another release. diff --git a/tests/fixtures/decode/arrays-nested.json b/tests/fixtures/decode/arrays-nested.json index 57ffe90..34d17d6 100644 --- a/tests/fixtures/decode/arrays-nested.json +++ b/tests/fixtures/decode/arrays-nested.json @@ -52,8 +52,8 @@ "specSection": "9.4" }, { - "name": "parses list items whose first field is a tabular array", - "input": "items[1]:\n - users[2]{id,name}:\n 1,Ada\n 2,Bob\n status: active", + "name": "parses list items whose first field is a tabular array, with and without sibling fields", + "input": "items[2]:\n - users[2]{id,name}:\n 1,Ada\n 2,Bob\n status: active\n - users[2]{id,name}:\n 3,Cy\n 4,Dee", "expected": { "items": [ { @@ -62,54 +62,33 @@ { "id": 2, "name": "Bob" } ], "status": "active" - } - ] - }, - "specSection": "10", - "note": "Canonical encoding: tabular header on hyphen line, rows at depth +2, sibling fields at depth +1" - }, - { - "name": "parses single-field list-item object with tabular array", - "input": "items[1]:\n - users[2]{id,name}:\n 1,Ada\n 2,Bob", - "expected": { - "items": [ + }, { "users": [ - { "id": 1, "name": "Ada" }, - { "id": 2, "name": "Bob" } + { "id": 3, "name": "Cy" }, + { "id": 4, "name": "Dee" } ] } ] }, - "specSection": "10" + "specSection": "10", + "note": "Canonical encoding: tabular header on hyphen line, rows at depth +2, sibling fields at depth +1" }, { - "name": "parses objects containing arrays (including empty arrays) in list form", + "name": "parses a legacy [0] header as a later field of a list-item object", "input": "items[1]:\n - name: Ada\n data[0]:", "expected": { - "items": [ - { "name": "Ada", "data": [] } - ] + "items": [{ "name": "Ada", "data": [] }] }, - "specSection": "9.4" + "specSection": "10" }, { - "name": "parses arrays of arrays within objects", + "name": "parses a list-form array as the first field of a list-item object", "input": "items[1]:\n - matrix[2]:\n - [2]: 1,2\n - [2]: 3,4\n name: grid", "expected": { - "items": [ - { "matrix": [[1, 2], [3, 4]], "name": "grid" } - ] - }, - "specSection": "9.2" - }, - { - "name": "parses nested arrays of primitives", - "input": "pairs[2]:\n - [2]: a,b\n - [2]: c,d", - "expected": { - "pairs": [["a", "b"], ["c", "d"]] + "items": [{ "matrix": [[1, 2], [3, 4]], "name": "grid" }] }, - "specSection": "9.2" + "specSection": "10" }, { "name": "parses quoted strings and mixed lengths in nested arrays", @@ -127,14 +106,6 @@ }, "specSection": "9.2" }, - { - "name": "parses mixed-length inner arrays", - "input": "pairs[2]:\n - [1]: 1\n - [2]: 2,3", - "expected": { - "pairs": [[1], [2, 3]] - }, - "specSection": "9.2" - }, { "name": "parses root-level primitive array inline", "input": "[5]: x,y,\"true\",true,10", @@ -193,25 +164,6 @@ }, "specSection": "9.4" }, - { - "name": "parses arrays mixing objects and arrays", - "input": "items[2]:\n - a: 1\n - [2]: 1,2", - "expected": { - "items": [{ "a": 1 }, [1, 2]] - }, - "specSection": "9.4" - }, - { - "name": "parses quoted key with list-form array", - "input": "\"x-items\"[2]:\n - id: 1\n - id: 2", - "expected": { - "x-items": [ - { "id": 1 }, - { "id": 2 } - ] - }, - "specSection": "9.4" - }, { "name": "accepts bare bracket pair as empty inner array list item", "input": "items[2]:\n - []\n - [1]: x", @@ -244,6 +196,90 @@ "shouldError": true, "specSection": "9.4", "minSpecVersion": "4.1" + }, + { + "name": "parses an array of arrays in list-item position", + "input": "[1]:\n - [1]:\n - [1]: 1", + "expected": [ + [ + [ + 1 + ] + ] + ], + "specSection": "9.4" + }, + { + "name": "parses an empty object as the first field of a list-item object", + "input": "items[1]:\n - a:\n b: 1", + "expected": { + "items": [ + { + "a": {}, + "b": 1 + } + ] + }, + "specSection": "10" + }, + { + "name": "parses a tabular array as a later field of a list-item object", + "input": "items[1]:\n - id: 1\n users[2]{id,name}:\n 1,Ada\n 2,Bob", + "expected": { + "items": [ + { + "id": 1, + "users": [ + { + "id": 1, + "name": "Ada" + }, + { + "id": 2, + "name": "Bob" + } + ] + } + ] + }, + "specSection": "10" + }, + { + "name": "parses quoted list item shaped like an inline array header", + "input": "xs[1]:\n - \"[2]: x\"", + "expected": { + "xs": [ + "[2]: x" + ] + }, + "specSection": "9.4" + }, + { + "name": "keeps list items under a legacy [0] header in non-strict mode", + "input": "a[0]:\n - x", + "expected": { + "a": [ + "x" + ] + }, + "options": { + "strict": false + }, + "specSection": "14.1", + "note": "A legacy [0] is a declared length, so its content is decoded (§14.1)", + "minSpecVersion": "4.1" + }, + { + "name": "parses list items shaped like headers without a colon as strings", + "input": "a[2]:\n - [2]\n - [1,2,3]", + "expected": { + "a": [ + "[2]", + "[1,2,3]" + ] + }, + "specSection": "6", + "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/decode/arrays-primitive.json b/tests/fixtures/decode/arrays-primitive.json index e09b87f..96978fe 100644 --- a/tests/fixtures/decode/arrays-primitive.json +++ b/tests/fixtures/decode/arrays-primitive.json @@ -3,22 +3,6 @@ "category": "decode", "description": "Primitive array decoding – inline arrays of strings, numbers, booleans, quoted strings", "tests": [ - { - "name": "parses string arrays inline", - "input": "tags[3]: reading,gaming,coding", - "expected": { - "tags": ["reading", "gaming", "coding"] - }, - "specSection": "9.1" - }, - { - "name": "parses number arrays inline", - "input": "nums[3]: 1,2,3", - "expected": { - "nums": [1, 2, 3] - }, - "specSection": "9.1" - }, { "name": "parses mixed primitive arrays inline", "input": "data[4]: x,y,true,10", @@ -100,60 +84,88 @@ "specSection": "9.1" }, { - "name": "parses quoted key containing brackets with inline array", - "input": "\"key[test]\"[3]: 1,2,3", + "name": "parses quoted key containing a colon with inline array", + "input": "\"a:b\"[2]: 1,2", "expected": { - "key[test]": [1, 2, 3] + "a:b": [1, 2] }, "specSection": "9.1" }, { - "name": "parses quoted key containing a colon with inline array", - "input": "\"a:b\"[2]: 1,2", + "name": "decodes canonical empty array key: []", + "input": "items: []", "expected": { - "a:b": [1, 2] + "items": [] }, "specSection": "9.1" }, { - "name": "parses quoted key with empty array", - "input": "\"x-custom\"[0]:", + "name": "parses quoted key containing escaped quotes and brackets with inline array", + "input": "\"has \\\"[2]\\\" in it\"[3]: 3,4,5", "expected": { - "x-custom": [] + "has \"[2]\" in it": [ + 3, + 4, + 5 + ] }, - "specSection": "9.1" + "specSection": "9.1", + "note": "An escaped quote does not close a quoted key, so the [2] inside it is not a bracket segment" }, { - "name": "parses quoted empty string key with empty array", - "input": "\"\"[0]:", + "name": "keeps inline values under a legacy [0] header in non-strict mode", + "input": "a[0]: 1,2", "expected": { - "": [] + "a": [ + 1, + 2 + ] }, - "specSection": "9.1" + "options": { + "strict": false + }, + "specSection": "14.1", + "note": "A legacy [0] is a declared length, so its content is decoded (§14.1)", + "minSpecVersion": "4.1" }, { - "name": "decodes canonical empty array key: []", - "input": "items: []", + "name": "keeps every inline value when the count mismatches in non-strict mode", + "input": "tags[1]: a,b", "expected": { - "items": [] + "tags": [ + "a", + "b" + ] }, - "specSection": "9.1" + "options": { + "strict": false + }, + "specSection": "14.1", + "minSpecVersion": "4.1" }, { - "name": "decodes canonical empty array with quoted key", - "input": "\"x-custom\": []", + "name": "decodes the inline element [] as a string, not an empty array", + "input": "a[1]: []", "expected": { - "x-custom": [] + "a": [ + "[]" + ] }, - "specSection": "9.1" + "specSection": "9.3" }, { - "name": "decodes canonical empty array with empty-string key", - "input": "\"\": []", + "name": "keeps a header whose length exceeds integer range in non-strict mode", + "input": "a[99999999999999999999]: 1", "expected": { - "": [] + "a": [ + 1 + ] }, - "specSection": "9.1" + "options": { + "strict": false + }, + "specSection": "6", + "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/decode/arrays-tabular.json b/tests/fixtures/decode/arrays-tabular.json index 9c18ed9..1f01652 100644 --- a/tests/fixtures/decode/arrays-tabular.json +++ b/tests/fixtures/decode/arrays-tabular.json @@ -25,17 +25,6 @@ }, "specSection": "9.3" }, - { - "name": "parses quoted colon in tabular row as data", - "input": "items[2]{id,note}:\n 1,\"a:b\"\n 2,\"c:d\"", - "expected": { - "items": [ - { "id": 1, "note": "a:b" }, - { "id": 2, "note": "c:d" } - ] - }, - "specSection": "9.3" - }, { "name": "parses quoted header keys in tabular arrays", "input": "items[2]{\"order:id\",\"full name\"}:\n 1,Ada\n 2,Bob", @@ -159,11 +148,11 @@ "specSection": "9.3" }, { - "name": "applies LWW for duplicate field names in non-strict mode", - "input": "items[1]{a,a}:\n 1,2", + "name": "applies LWW for duplicate field names, bare or grouped, in non-strict mode", + "input": "items[1]{a,a,b,b{x}}:\n 1,2,3,4", "expected": { "items": [ - { "a": 2 } + { "a": 2, "b": { "x": 4 } } ] }, "options": { @@ -172,19 +161,6 @@ "specSection": "14.3", "note": "Duplicate field names yield duplicate sibling keys per row; non-strict resolves last-write-wins" }, - { - "name": "applies LWW when a bare field and a nested group share a name in non-strict mode", - "input": "items[1]{a,a{x}}:\n 1,2", - "expected": { - "items": [ - { "a": { "x": 2 } } - ] - }, - "options": { - "strict": false - }, - "specSection": "14.3" - }, { "name": "accepts a field name outside the encoder unquoted-key pattern", "input": "items[1]{2key}:\n 1", @@ -217,6 +193,114 @@ "specSection": "14.1", "note": "A declared [N] never truncates a scope", "minSpecVersion": "4.1" + }, + { + "name": "parses quoted leading tabular cell shaped like an inline array header", + "input": "rows[1]{note,id}:\n \"[2]: x\",1", + "expected": { + "rows": [ + { + "note": "[2]: x", + "id": 1 + } + ] + }, + "specSection": "9.3" + }, + { + "name": "trims spaces around field names", + "input": "items[1]{ a , b }:\n 1,2", + "expected": { + "items": [ + { + "a": 1, + "b": 2 + } + ] + }, + "specSection": "12", + "minSpecVersion": "4.1" + }, + { + "name": "materializes a nested field group without cells as an empty object in non-strict mode", + "input": "items[1]{a,n{x}}:\n 1", + "expected": { + "items": [ + { + "a": 1, + "n": {} + } + ] + }, + "options": { + "strict": false + }, + "specSection": "14.1", + "minSpecVersion": "4.1" + }, + { + "name": "drops surplus cells when the width mismatches in non-strict mode", + "input": "items[1]{a}:\n 1,2", + "expected": { + "items": [ + { + "a": 1 + } + ] + }, + "options": { + "strict": false + }, + "specSection": "14.1", + "minSpecVersion": "4.1" + }, + { + "name": "parses unquoted colon after the delimiter in tabular row as data", + "input": "items[1]{id,note}:\n 1,a:b", + "expected": { + "items": [ + { + "id": 1, + "note": "a:b" + } + ] + }, + "specSection": "9.3" + }, + { + "name": "ignores a brace inside a mid-name quote span in a field list", + "input": "a[1]{x\"}\"y}:\n 1", + "expected": { + "a": [ + { + "x\"}\"y": 1 + } + ] + }, + "specSection": "6", + "minSpecVersion": "4.2" + }, + { + "name": "parses a quoted empty field name", + "input": "[2]{\"\"}:\n 1\n 2", + "expected": [ + { + "": 1 + }, + { + "": 2 + } + ], + "specSection": "6", + "minSpecVersion": "4.2" + }, + { + "name": "unescapes a quoted field name", + "input": "a[1]{\"x\\\"y\"}:\n 1", + "expected": { + "a": [{ "x\"y": 1 }] + }, + "specSection": "6" } ] } diff --git a/tests/fixtures/decode/blank-lines.json b/tests/fixtures/decode/blank-lines.json index b8eb4f9..a25a5a9 100644 --- a/tests/fixtures/decode/blank-lines.json +++ b/tests/fixtures/decode/blank-lines.json @@ -5,7 +5,7 @@ "tests": [ { "name": "throws on blank line inside list array", - "input": "items[3]:\n - a\n\n - b\n - c", + "input": "items[2]:\n - a\n\n - b", "expected": null, "shouldError": true, "options": { @@ -55,7 +55,7 @@ }, { "name": "throws on blank line in nested list array", - "input": "outer[2]:\n - inner[2]:\n - a\n\n - b\n - x", + "input": "outer[2]:\n - inner[2]:\n - a\n\n - b\n - x", "expected": null, "shouldError": true, "options": { @@ -76,7 +76,7 @@ "specSection": "12" }, { - "name": "accepts whitespace-only line at non-multiple indent as blank in strict mode", + "name": "accepts a spaces-only line at non-multiple indent as blank in strict mode", "input": "a: 1\n \nb: 2", "expected": { "a": 1, @@ -137,7 +137,7 @@ }, { "name": "ignores blank lines inside list array when strict=false", - "input": "items[3]:\n - a\n\n - b\n - c", + "input": "items[3]:\n - a\n\n - b\n\n\n - c", "expected": { "items": ["a", "b", "c"] }, @@ -160,17 +160,6 @@ }, "specSection": "12" }, - { - "name": "ignores multiple blank lines in arrays when strict=false", - "input": "items[2]:\n - a\n\n\n - b", - "expected": { - "items": ["a", "b"] - }, - "options": { - "strict": false - }, - "specSection": "12" - }, { "name": "throws on blank line between list items after nested tabular rows", "input": "outer[2]:\n - inner[2]{a}:\n 1\n 2\n\n - x", @@ -179,13 +168,6 @@ "specSection": "14.2", "note": "The blank is after the inner scope's last row but inside the outer array's span (\u00a712)" }, - { - "name": "throws on blank line between a list item's fields", - "input": "items[2]:\n - a: 1\n\n b: 2\n - x", - "expected": null, - "shouldError": true, - "specSection": "14.2" - }, { "name": "throws on blank line inside the last list item's fields", "input": "items[1]:\n - a: 1\n\n b: 2", @@ -211,6 +193,43 @@ "input": "m[2:]{v}:\n\n a: 1\n b: 2", "expected": { "m": { "a": { "v": 1 }, "b": { "v": 2 } } }, "specSection": "12" + }, + { + "name": "throws on blank line between a nested header and its first row inside a list item", + "input": "outer[2]:\n - inner[1]{a}:\n\n 1\n - x", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "The blank is between the inner header and its first row but inside the outer array's span (§12)" + }, + { + "name": "throws on blank line between list items after a nested object", + "input": "a[2]:\n - x:\n y: 1\n\n - 2", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "The blank is after the nested object's content but inside the outer array's span (§12)" + }, + { + "name": "throws on a tab-only line, which is not blank", + "input": "a: 1\n\t\nb: 2", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "minSpecVersion": "4.2" + }, + { + "name": "treats a tab-only line as blank in non-strict mode", + "input": "a: 1\n\t\nb: 2", + "expected": { + "a": 1, + "b": 2 + }, + "options": { + "strict": false + }, + "specSection": "12", + "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/decode/comments.json b/tests/fixtures/decode/comments.json index e351082..5db101b 100644 --- a/tests/fixtures/decode/comments.json +++ b/tests/fixtures/decode/comments.json @@ -4,8 +4,8 @@ "description": "Comment lines – full-line decode-side comments stripped before structural interpretation", "tests": [ { - "name": "strips comment line between tabular rows without terminating rows", - "input": "items[2]{id,name}:\n 1,Ada\n # note\n 2,Bob", + "name": "strips comment lines after the header and between tabular rows", + "input": "items[2]{id,name}:\n # leading\n 1,Ada\n # note\n 2,Bob", "expected": { "items": [ { @@ -19,22 +19,7 @@ ] }, "specSection": "5.1", - "note": "The comment is not counted as a row and does not end the tabular scope" - }, - { - "name": "strips comment line between header and first row", - "input": "items[2]{id}:\n # leading\n 1\n 2", - "expected": { - "items": [ - { - "id": 1 - }, - { - "id": 2 - } - ] - }, - "specSection": "5.1" + "note": "Comments are not counted as rows and do not end the tabular scope" }, { "name": "strips comment line whose indentation is not a multiple of indentSize", @@ -88,12 +73,6 @@ "specSection": "5.1", "note": "Comment text is discarded without interpretation; the quote never reaches the tokenizer" }, - { - "name": "decodes document of only comments to empty object", - "input": "# only\n# comments", - "expected": {}, - "specSection": "5.1" - }, { "name": "decodes comments and blank lines only to empty object", "input": "\n# c\n\n# d\n", @@ -112,24 +91,20 @@ "input": "items[2]{id}:\n #1\n 2", "expected": null, "shouldError": true, - "specSection": "14.1", + "specSection": "5.1", "note": "The first cell makes the whole line a comment; only one row remains for [2]" }, { - "name": "drops hash-leading row silently in non-strict mode", + "name": "strips a hash-leading row as a comment in non-strict mode", "input": "items[2]{id}:\n #1\n 2", "expected": { - "items": [ - { - "id": 2 - } - ] + "items": [{ "id": 2 }] }, "options": { "strict": false }, "specSection": "5.1", - "note": "A hash-leading first cell makes the whole line a comment; non-strict mode drops the row silently" + "note": "Comment removal applies in both modes; the resulting count mismatch is not an error when strict=false (§14.1)" }, { "name": "throws on tab-indented hash line, which is not a comment", @@ -185,33 +160,13 @@ "note": "\"#\" is not the line's first non-space character; comments are full-line only" }, { - "name": "parses hash after key-value colon as string value", - "input": "note: #x", - "expected": { - "note": "#x" - }, - "specSection": "5.1", - "note": "No inline comments; the unquoted token decodes as the string #x" - }, - { - "name": "parses hash mid-value as data, not a trailing comment", - "input": "msg: hello # world", + "name": "parses hash after a key-value colon or mid-value as data, not an inline comment", + "input": "note: #x\nmsg: hello # world", "expected": { + "note": "#x", "msg": "hello # world" }, "specSection": "5.1" - }, - { - "name": "round-trips quoted hash-leading values in field and inline array positions", - "input": "tag: \"#x\"\nlist[2]: \"#a\",b", - "expected": { - "tag": "#x", - "list": [ - "#a", - "b" - ] - }, - "specSection": "7.2" } ] } diff --git a/tests/fixtures/decode/delimiters.json b/tests/fixtures/decode/delimiters.json index d12802a..45b01d7 100644 --- a/tests/fixtures/decode/delimiters.json +++ b/tests/fixtures/decode/delimiters.json @@ -58,7 +58,7 @@ "specSection": "11" }, { - "name": "parses nested arrays inside list items with default comma delimiter", + "name": "parses nested arrays inside list items with default comma delimiter when parent uses tab", "input": "items[1\t]:\n - tags[3]: a,b,c", "expected": { "items": [{ "tags": ["a", "b", "c"] }] @@ -66,10 +66,10 @@ "specSection": "11" }, { - "name": "parses nested arrays inside list items with default comma delimiter when parent uses pipe", - "input": "items[1|]:\n - tags[3]: a,b,c", + "name": "parses a keyless inner array without a delimiter symbol as comma when parent uses pipe", + "input": "pairs[1|]:\n - [2]: a,b", "expected": { - "items": [{ "tags": ["a", "b", "c"] }] + "pairs": [["a", "b"]] }, "specSection": "11" }, @@ -87,19 +87,19 @@ }, { "name": "parses root-level array of objects with tab delimiter", - "input": "[2\t]{id}:\n 1\n 2", - "expected": [{ "id": 1 }, { "id": 2 }], + "input": "[1\t]{a\tb}:\n 1\t2", + "expected": [{ "a": 1, "b": 2 }], "specSection": "11" }, { "name": "parses root-level array of objects with pipe delimiter", - "input": "[2|]{id}:\n 1\n 2", - "expected": [{ "id": 1 }, { "id": 2 }], + "input": "[1|]{a|b}:\n 1|2", + "expected": [{ "a": 1, "b": 2 }], "specSection": "11" }, { "name": "parses values containing tab delimiter when quoted", - "input": "items[3\t]: a\t\"b\\tc\"\td", + "input": "items[3\t]: a\t\"b\tc\"\td", "expected": { "items": ["a", "b\tc", "d"] }, @@ -161,10 +161,10 @@ "note": "Object field values are parsed as the whole post-colon token, never split on a delimiter" }, { - "name": "does not split object values on commas inside a tab-delimited scope", - "input": "items[2\t]:\n - status: a,b\n - status: c,d", + "name": "does not split a list-item field value on the active delimiter", + "input": "items[1|]:\n - status: a|b", "expected": { - "items": [{ "status": "a,b" }, { "status": "c,d" }] + "items": [{ "status": "a|b" }] }, "specSection": "11.2", "note": "The active delimiter splits row and inline cells, never object field values" @@ -177,54 +177,6 @@ }, "specSection": "11" }, - { - "name": "parses nested array values containing pipe delimiter", - "input": "pairs[1|]:\n - [2|]: a|\"b|c\"", - "expected": { - "pairs": [["a", "b|c"]] - }, - "specSection": "11" - }, - { - "name": "parses nested array values containing tab delimiter", - "input": "pairs[1\t]:\n - [2\t]: a\t\"b\\tc\"", - "expected": { - "pairs": [["a", "b\tc"]] - }, - "specSection": "11" - }, - { - "name": "preserves quoted ambiguity with pipe delimiter", - "input": "items[3|]: \"true\"|\"42\"|\"-3.14\"", - "expected": { - "items": ["true", "42", "-3.14"] - }, - "specSection": "11" - }, - { - "name": "preserves quoted ambiguity with tab delimiter", - "input": "items[3\t]: \"true\"\t\"42\"\t\"-3.14\"", - "expected": { - "items": ["true", "42", "-3.14"] - }, - "specSection": "11" - }, - { - "name": "parses structural-looking strings when quoted with pipe delimiter", - "input": "items[3|]: \"[5]\"|\"{key}\"|\"- item\"", - "expected": { - "items": ["[5]", "{key}", "- item"] - }, - "specSection": "11" - }, - { - "name": "parses structural-looking strings when quoted with tab delimiter", - "input": "items[3\t]: \"[5]\"\t\"{key}\"\t\"- item\"", - "expected": { - "items": ["[5]", "{key}", "- item"] - }, - "specSection": "11" - }, { "name": "parses tabular headers with keys containing the active delimiter", "input": "items[2|]{\"a|b\"}:\n 1\n 2", @@ -235,15 +187,35 @@ }, { "name": "falls through to a key-value line on a header delimiter mismatch", - "input": "a[2|]{x,y}: 1|2", + "input": "a[0|]{x,y}:", "expected": { - "a[2|]{x,y}": "1|2" + "a[0|]{x,y}": {} }, "options": { "strict": false }, "specSection": "6", "minSpecVersion": "4.1" + }, + { + "name": "treats a delimiter inside a mid-token quote span as quoted", + "input": "k[2]: a\"b,c\",d", + "expected": { + "k": [ + "a\"b,c\"", + "d" + ] + }, + "specSection": "5.2", + "minSpecVersion": "4.2" + }, + { + "name": "parses a nested tabular header with its own comma delimiter inside a pipe list", + "input": "items[1|]:\n - t[1]{a,b}:\n 1,2", + "expected": { + "items": [{ "t": [{ "a": 1, "b": 2 }] }] + }, + "specSection": "11.2" } ] } diff --git a/tests/fixtures/decode/indentation-errors.json b/tests/fixtures/decode/indentation-errors.json index 96097d0..378f080 100644 --- a/tests/fixtures/decode/indentation-errors.json +++ b/tests/fixtures/decode/indentation-errors.json @@ -26,8 +26,8 @@ "specSection": "14.2" }, { - "name": "throws on non-multiple indentation with custom indentSize 4 (3 spaces)", - "input": "a:\n b: 1", + "name": "throws on 2-space indentation with custom indentSize 4", + "input": "a:\n b: 1", "expected": null, "shouldError": true, "options": { @@ -114,27 +114,12 @@ "specSection": "12" }, { - "name": "accepts non-multiple indentation when strict=false", - "input": "a:\n b: 1", + "name": "accepts non-multiple indentation at nested depths when strict=false", + "input": "a:\n b:\n c: 1\n d: 2", "expected": { "a": { - "b": 1 - } - }, - "options": { - "indentSize": 2, - "strict": false - }, - "specSection": "12" - }, - { - "name": "accepts deeply nested non-multiples when strict=false", - "input": "a:\n b:\n c: 1", - "expected": { - "a": { - "b": { - "c": 1 - } + "b": { "c": 1 }, + "d": 2 } }, "options": { @@ -148,7 +133,7 @@ "input": "a:\n b: 1", "expected": null, "shouldError": true, - "specSection": "14.2", + "specSection": "8", "note": "Four leading spaces are a multiple of indentSize 2, but depth 2 directly under depth 0 skips a level" }, { @@ -156,22 +141,21 @@ "input": "a:\n b:\n c: 1", "expected": null, "shouldError": true, - "specSection": "14.2" + "specSection": "8" }, { - "name": "throws on over-indented line after a primitive field", - "input": "a: 1\n b: 2", + "name": "throws on an over-indented line under a primitive list item", + "input": "items[2]:\n - a\n b: 1\n - c", "expected": null, "shouldError": true, - "specSection": "14.2", - "note": "A primitive field opens no scope, so the depth-2 line belongs to none and must not be silently discarded" + "specSection": "8" }, { "name": "throws on a line indented one level under a primitive field", "input": "a: 1\n b: 2", "expected": null, "shouldError": true, - "specSection": "14.2", + "specSection": "8", "note": "Even a single extra level is over-indentation when the preceding line did not open a scope" }, { @@ -179,7 +163,7 @@ "input": "a:\n b: 1\n c: 2", "expected": null, "shouldError": true, - "specSection": "14.2", + "specSection": "8", "note": "The depth-3 line follows a primitive field at depth 1; it belongs to no scope" }, { @@ -187,15 +171,127 @@ "input": "u[1]{a}:\n 1\n junk: 9", "expected": null, "shouldError": true, - "specSection": "14.2" + "specSection": "8" }, { - "name": "throws on orphan scalar line under a primitive field", + "name": "throws on orphan scalar line under a primitive field in non-strict mode", "input": "a: 1\n hello", "expected": null, "shouldError": true, - "specSection": "14.2", - "note": "A scalar line is valid only at root primitive position" + "options": { + "strict": false + }, + "specSection": "8", + "note": "The non-strict skip for over-indented lines excludes scalar lines (§5.2)", + "minSpecVersion": "4.1" + }, + { + "name": "throws on an indented first line", + "input": " a: 1", + "expected": null, + "shouldError": true, + "specSection": "8", + "minSpecVersion": "4.2" + }, + { + "name": "throws on a hyphen-leading bare token under a primitive field in non-strict mode", + "input": "a: 1\n - x", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "5.2", + "minSpecVersion": "4.2" + }, + { + "name": "adopts the depth of a jumped first line in non-strict mode", + "input": "a:\n b: 1\n c: 2\n d: 3", + "expected": { + "a": { + "b": 1, + "d": 3 + } + }, + "options": { + "strict": false + }, + "specSection": "8", + "minSpecVersion": "4.2" + }, + { + "name": "skips a line above the adopted entry depth in non-strict mode", + "input": "m[1:]{a}:\n x: 1\n y: 2", + "expected": { + "m": { + "x": { + "a": 1 + } + } + }, + "options": { + "strict": false + }, + "specSection": "8", + "minSpecVersion": "4.2" + }, + { + "name": "skips a line above the adopted row depth in non-strict mode", + "input": "rows[2]{a}:\n 1\n x: 9\n 2", + "expected": { + "rows": [ + { + "a": 1 + }, + { + "a": 2 + } + ] + }, + "options": { + "strict": false + }, + "specSection": "8", + "minSpecVersion": "4.2" + }, + { + "name": "skips an over-indented line under a primitive list item in non-strict mode", + "input": "items[2]:\n - a\n b: 1\n - c", + "expected": { + "items": [ + "a", + "c" + ] + }, + "options": { + "strict": false + }, + "specSection": "8", + "minSpecVersion": "4.2" + }, + { + "name": "skips an over-indented line under a list-item field in non-strict mode", + "input": "items[1]:\n - a: 1\n deep: 1\n b: 2", + "expected": { + "items": [ + { + "a": 1, + "b": 2 + } + ] + }, + "options": { + "strict": false + }, + "specSection": "8", + "minSpecVersion": "4.2" + }, + { + "name": "throws on a depth jump to the first tabular row", + "input": "rows[1]{a}:\n 1", + "expected": null, + "shouldError": true, + "specSection": "8" } ] } diff --git a/tests/fixtures/decode/numbers.json b/tests/fixtures/decode/numbers.json index 22ac3f1..f35ee55 100644 --- a/tests/fixtures/decode/numbers.json +++ b/tests/fixtures/decode/numbers.json @@ -12,45 +12,24 @@ "specSection": "4" }, { - "name": "parses negative number with positive exponent", - "input": "value: -1E+03", + "name": "parses exponent forms in either case with optional exponent sign", + "input": "a: -1E+03\nb: 2.5e2\nc: 3E-02", "expected": { - "value": -1000 - }, - "specSection": "4" - }, - { - "name": "parses lowercase exponent", - "input": "value: 2.5e2", - "expected": { - "value": 250 - }, - "specSection": "4" - }, - { - "name": "parses uppercase exponent with negative sign", - "input": "value: 3E-02", - "expected": { - "value": 0.03 + "a": -1000, + "b": 250, + "c": 0.03 }, "specSection": "4" }, { "name": "parses negative zero as zero", - "input": "value: -0", + "input": "a: -0\nb: -0.0", "expected": { - "value": 0 + "a": 0, + "b": 0 }, "specSection": "4", - "note": "Negative zero decodes to 0; most host environments do not distinguish -0 from 0" - }, - { - "name": "parses negative zero with fractional part", - "input": "value: -0.0", - "expected": { - "value": 0 - }, - "specSection": "4" + "note": "Most host environments do not distinguish -0 from 0" }, { "name": "parses array with mixed numeric forms", @@ -58,8 +37,7 @@ "expected": { "nums": [42, -1000, 1.5, 0, 250] }, - "specSection": "4", - "note": "Decoders normalize all numeric forms to host numeric values" + "specSection": "4" }, { "name": "treats leading zero as string not number", @@ -69,22 +47,6 @@ }, "specSection": "4" }, - { - "name": "parses very small exponent", - "input": "value: 1e-10", - "expected": { - "value": 0.0000000001 - }, - "specSection": "4" - }, - { - "name": "parses integer with positive exponent", - "input": "value: 5E+00", - "expected": { - "value": 5 - }, - "specSection": "4" - }, { "name": "parses zero with exponent as number", "input": "value: 0e1", @@ -93,50 +55,18 @@ }, "specSection": "4" }, - { - "name": "parses negative zero with exponent as number", - "input": "value: -0e1", - "expected": { - "value": 0 - }, - "specSection": "4" - }, { "name": "parses exponent notation", "input": "1e6", "expected": 1000000, "specSection": "4" }, - { - "name": "parses exponent notation with uppercase E", - "input": "1E+6", - "expected": 1000000, - "specSection": "4" - }, - { - "name": "parses negative exponent notation", - "input": "-1e-3", - "expected": -0.001, - "specSection": "4" - }, { "name": "treats unquoted leading-zero number as string", "input": "05", "expected": "05", "specSection": "4" }, - { - "name": "treats unquoted multi-leading-zero as string", - "input": "007", - "expected": "007", - "specSection": "4" - }, - { - "name": "treats unquoted octal-like as string", - "input": "0123", - "expected": "0123", - "specSection": "4" - }, { "name": "treats leading-zeros in array as strings", "input": "nums[3]: 05,007,0123", @@ -150,38 +80,15 @@ "specSection": "4" }, { - "name": "treats negative leading-zeros in array as strings", - "input": "nums[2]: -05,-007", - "expected": { - "nums": ["-05", "-007"] - }, - "specSection": "4" - }, - { - "name": "treats bare fraction without integer part as string", - "input": "value: .5", - "expected": { - "value": ".5" - }, - "specSection": "4", - "note": "Outside the number grammar, so a host parser would read 0.5 and corrupt the round-trip" - }, - { - "name": "treats integer with trailing dot as string", - "input": "value: 1.", - "expected": { - "value": "1." - }, - "specSection": "4" - }, - { - "name": "treats leading-plus integer as string", - "input": "value: +1", + "name": "treats bare-fraction, trailing-dot, and leading-plus tokens as strings", + "input": "a: .5\nb: 1.\nc: +1", "expected": { - "value": "+1" + "a": ".5", + "b": "1.", + "c": "+1" }, "specSection": "4", - "note": "The grammar admits only an optional leading minus, so +1 stays a string" + "note": "Outside the number grammar, so a host number parser would corrupt the round-trip" }, { "name": "treats leading-plus tokens in array as strings", @@ -192,15 +99,6 @@ "specSection": "4", "note": "The grammar admits only an optional leading minus, so leading-plus cells stay strings" }, - { - "name": "parses positive exponent sign as number", - "input": "value: 1e+2", - "expected": { - "value": 100 - }, - "specSection": "4", - "note": "The plus sign is inside the exponent, which the grammar allows" - }, { "name": "treats Infinity and NaN tokens as strings", "input": "a: Infinity\nb: -Infinity\nc: NaN", @@ -220,6 +118,14 @@ }, "specSection": "4", "note": "Host parsers that accept these forms have a wider grammar than the spec permits" + }, + { + "name": "treats non-ASCII digits as a string", + "input": "value: ٤٢", + "expected": { + "value": "٤٢" + }, + "specSection": "4" } ] } diff --git a/tests/fixtures/decode/objects-keyed.json b/tests/fixtures/decode/objects-keyed.json index 3e2e05d..848030a 100644 --- a/tests/fixtures/decode/objects-keyed.json +++ b/tests/fixtures/decode/objects-keyed.json @@ -36,21 +36,14 @@ "specSection": "9.5" }, { - "name": "parses keyed headers with the pipe delimiter", - "input": "servers[2:|]{host|port}:\n alpha: a|8080\n beta: b|9090", + "name": "parses keyed headers with the pipe and tab delimiters", + "input": "pipe[2:|]{host|port}:\n alpha: a|8080\n beta: b|9090\ntab[2:\t]{host\tport}:\n alpha: a\t8080\n beta: b\t9090", "expected": { - "servers": { + "pipe": { "alpha": { "host": "a", "port": 8080 }, "beta": { "host": "b", "port": 9090 } - } - }, - "specSection": "9.5" - }, - { - "name": "parses keyed headers with the tab delimiter", - "input": "servers[2:\t]{host\tport}:\n alpha: a\t8080\n beta: b\t9090", - "expected": { - "servers": { + }, + "tab": { "alpha": { "host": "a", "port": 8080 }, "beta": { "host": "b", "port": 9090 } } @@ -93,27 +86,16 @@ "note": "A keyed scope ends only by dedent or end of input" }, { - "name": "treats key-value-shaped lines at entry depth as entry rows", - "input": "m[2:]{x}:\n total: 5\n other: 6", - "expected": { - "m": { - "total": { "x": 5 }, - "other": { "x": 6 } - } - }, - "specSection": "9.5", - "note": "Opposite polarity to §9.3: in a tabular array, a colon-before-delimiter line at row depth ends the rows; in a keyed scope it is an entry" - }, - { - "name": "splits an entry row at its first unquoted colon before delimiter splitting", - "input": "m[1:]{v}:\n k[2]: 5", + "name": "splits an entry row at its first unquoted colon before delimiter or header parsing", + "input": "m[2:]{v}:\n a,b: 5\n k[2]: 6", "expected": { "m": { - "k[2]": { "v": 5 } + "a,b": { "v": 5 }, + "k[2]": { "v": 6 } } }, "specSection": "9.5", - "note": "The token before the first unquoted colon is the entry key, accepted as a literal key per §7.4 even though an encoder would quote it" + "note": "The entry key is everything before the first unquoted colon, taken as a literal key per §7.4 even though an encoder would quote it" }, { "name": "decodes the cell token [] as a string, not an empty array", @@ -136,16 +118,6 @@ "specSection": "9.5", "note": "Encoders never emit keyed headers for fewer than two entries; decoders accept any N with count checks" }, - { - "name": "accepts a single entry row", - "input": "m[1:]{v}:\n a: 1", - "expected": { - "m": { - "a": { "v": 1 } - } - }, - "specSection": "9.5" - }, { "name": "parses quoted cells containing the active delimiter", "input": "m[2:]{t}:\n a: \"x,y\"\n b: z", @@ -225,6 +197,18 @@ }, "specSection": "9.5", "minSpecVersion": "4.1" + }, + { + "name": "decodes every entry row beyond the declared count in non-strict mode", + "input": "m[1:]{v}:\n a: 1\n b: 2", + "expected": { + "m": { "a": { "v": 1 }, "b": { "v": 2 } } + }, + "options": { + "strict": false + }, + "specSection": "14.1", + "minSpecVersion": "4.1" } ] } diff --git a/tests/fixtures/decode/objects.json b/tests/fixtures/decode/objects.json index 1e42dbf..bca97f8 100644 --- a/tests/fixtures/decode/objects.json +++ b/tests/fixtures/decode/objects.json @@ -5,25 +5,17 @@ "tests": [ { "name": "parses objects with primitive values", - "input": "id: 123\nname: Ada\nactive: true", + "input": "id: 123\nname: Ada\nactive: true\nvalue: null", "expected": { "id": 123, "name": "Ada", - "active": true - }, - "specSection": "8" - }, - { - "name": "parses null values in objects", - "input": "id: 123\nvalue: null", - "expected": { - "id": 123, + "active": true, "value": null }, "specSection": "8" }, { - "name": "parses empty nested object header", + "name": "decodes a bare key: as an empty object, not an empty array", "input": "user:", "expected": { "user": {} @@ -31,12 +23,13 @@ "specSection": "8" }, { - "name": "bare key: with no children decodes as empty object, not array", - "input": "matches:", + "name": "decodes a bare key followed by trailing spaces as an empty object", + "input": "matches: ", "expected": { "matches": {} }, - "specSection": "8" + "specSection": "12", + "minSpecVersion": "4.1" }, { "name": "applies last-write-wins for duplicate sibling keys in non-strict mode", @@ -55,7 +48,7 @@ "expected": { "note": "a:b" }, - "specSection": "8" + "specSection": "4" }, { "name": "parses quoted object value with comma", @@ -63,7 +56,7 @@ "expected": { "note": "a,b" }, - "specSection": "8" + "specSection": "4" }, { "name": "parses quoted object value with newline escape", @@ -87,15 +80,7 @@ "expected": { "text": " padded " }, - "specSection": "8" - }, - { - "name": "parses quoted object value with only spaces", - "input": "text: \" \"", - "expected": { - "text": " " - }, - "specSection": "8" + "specSection": "4" }, { "name": "parses quoted string value that looks like true", @@ -103,7 +88,7 @@ "expected": { "v": "true" }, - "specSection": "8" + "specSection": "4" }, { "name": "parses quoted string value that looks like integer", @@ -111,24 +96,15 @@ "expected": { "v": "42" }, - "specSection": "8" + "specSection": "4" }, { - "name": "parses quoted string value that looks like negative decimal", - "input": "v: \"-7.5\"", - "expected": { - "v": "-7.5" - }, - "specSection": "8" - }, - { - "name": "parses quoted object value shaped like an inline array header", - "input": "a: \"[2]: x\"", + "name": "parses an unquoted field value starting with a bracket segment as a string", + "input": "a: [2]: x", "expected": { "a": "[2]: x" }, - "specSection": "8", - "note": "Quoted value is opaque (§4); the bracket-and-colon shape inside the quotes is data, not an array header (§6)" + "specSection": "5.2" }, { "name": "parses quoted object value shaped like a count-matching inline array header", @@ -136,7 +112,7 @@ "expected": { "a": "[1]: x" }, - "specSection": "8", + "specSection": "4", "note": "Quoting makes the value opaque (§4), so it is never parsed as a header – the matching count is irrelevant because a quoted scalar is never a header candidate" }, { @@ -145,7 +121,7 @@ "expected": { "key": "foo [2]: bar" }, - "specSection": "6", + "specSection": "5.2", "note": "An unquoted key cannot contain a colon, so the first colon ends the key; the bracket-and-colon shape in the remainder is value text, not an array header" }, { @@ -154,7 +130,7 @@ "expected": { "a": "b [2]{x,y}: 1,2" }, - "specSection": "6", + "specSection": "5.2", "note": "An unquoted key cannot contain a colon, so the first colon ends the key; the tabular-header shape in the remainder is value text, not an array header" }, { @@ -163,7 +139,7 @@ "expected": { "content": "accept (process.argv[2]) and greet: name" }, - "specSection": "8" + "specSection": "5.2" }, { "name": "parses quoted key with colon", @@ -171,7 +147,7 @@ "expected": { "order:id": 7 }, - "specSection": "8" + "specSection": "7.4" }, { "name": "decodes \\uXXXX in quoted key (U+0004 control character)", @@ -190,24 +166,26 @@ "specSection": "7.1" }, { - "name": "parses quoted key with brackets", - "input": "\"[index]\": 5", + "name": "parses quoted keys with brackets and braces", + "input": "\"[index]\": 5\n\"{key}\": 6", "expected": { - "[index]": 5 + "[index]": 5, + "{key}": 6 }, - "specSection": "8" + "specSection": "7.4" }, { - "name": "treats extra brackets after valid array segment as literal key (non-strict)", - "input": "foo[1][bar]: 10", + "name": "treats content between bracket segment and colon as literal key (non-strict)", + "input": "foo[1][bar]: 10\nfoo[2]extra: a,b", "options": { "strict": false }, "expected": { - "foo[1][bar]": 10 + "foo[1][bar]": 10, + "foo[2]extra": "a,b" }, "specSection": "6", - "note": "Non-whitespace [bar] between ] and : prevents array header interpretation; non-strict fall-through produces a literal key not constrained by §7.3" + "note": "Non-whitespace content between ] and : (a second bracket group or plain text) prevents array header interpretation; non-strict fall-through produces a literal key not constrained by §7.3" }, { "name": "treats bracket segment without a length as literal key (non-strict)", @@ -233,41 +211,22 @@ "specSection": "6", "note": "[bar] fails integer parsing; non-strict fall-through produces a literal key not constrained by §7.3" }, - { - "name": "treats text between bracket segment and colon as literal key (non-strict)", - "input": "foo[2]extra: a,b", - "options": { - "strict": false - }, - "expected": { - "foo[2]extra": "a,b" - }, - "specSection": "6", - "note": "Non-whitespace content between ] and : prevents array header interpretation; non-strict fall-through produces a literal key" - }, - { - "name": "parses quoted key with braces", - "input": "\"{key}\": 5", - "expected": { - "{key}": 5 - }, - "specSection": "8" - }, { "name": "parses quoted key with comma", "input": "\"a,b\": 1", "expected": { "a,b": 1 }, - "specSection": "8" + "specSection": "7.4" }, { - "name": "parses quoted key with spaces", - "input": "\"full name\": Ada", + "name": "parses quoted keys with inner, leading, and trailing spaces", + "input": "\"full name\": Ada\n\" a \": 1", "expected": { - "full name": "Ada" + "full name": "Ada", + " a ": 1 }, - "specSection": "8" + "specSection": "7.4" }, { "name": "parses quoted key with leading hyphen", @@ -275,15 +234,7 @@ "expected": { "-lead": 1 }, - "specSection": "8" - }, - { - "name": "parses quoted key with leading and trailing spaces", - "input": "\" a \": 1", - "expected": { - " a ": 1 - }, - "specSection": "8" + "specSection": "7.4" }, { "name": "parses quoted numeric key", @@ -291,7 +242,7 @@ "expected": { "123": "x" }, - "specSection": "8" + "specSection": "7.4" }, { "name": "parses quoted empty string key", @@ -299,7 +250,7 @@ "expected": { "": 1 }, - "specSection": "8" + "specSection": "7.4" }, { "name": "parses dotted keys as identifiers", @@ -310,34 +261,11 @@ "specSection": "8" }, { - "name": "parses underscore-prefixed keys", - "input": "_private: 1", - "expected": { - "_private": 1 - }, - "specSection": "8" - }, - { - "name": "parses underscore-containing keys", - "input": "user_name: 1", - "expected": { - "user_name": 1 - }, - "specSection": "8" - }, - { - "name": "unescapes newline in key", - "input": "\"line\\nbreak\": 1", - "expected": { - "line\nbreak": 1 - }, - "specSection": "7.1" - }, - { - "name": "unescapes tab in key", - "input": "\"tab\\there\": 2", + "name": "unescapes newline and quotes in keys", + "input": "\"line\\nbreak\": 1\n\"he said \\\"hi\\\"\": 2", "expected": { - "tab\there": 2 + "line\nbreak": 1, + "he said \"hi\"": 2 }, "specSection": "7.1" }, @@ -349,14 +277,6 @@ }, "specSection": "7.1" }, - { - "name": "unescapes quotes in key", - "input": "\"he said \\\"hi\\\"\": 1", - "expected": { - "he said \"hi\"": 1 - }, - "specSection": "7.1" - }, { "name": "parses deeply nested objects with indentation", "input": "a:\n b:\n c: deep", @@ -433,22 +353,15 @@ "note": "The first unquoted colon ends the key; the entire post-colon token is the value (§11.2)" }, { - "name": "materializes __proto__ as an ordinary own key", - "input": "__proto__: polluted", - "expected": { - "__proto__": "polluted" - }, - "specSection": "15", - "note": "Decoding MUST NOT mutate the host object model, so the decoded object carries __proto__ as an own entry" - }, - { - "name": "materializes constructor and prototype as ordinary own keys", - "input": "constructor: 1\nprototype: 2", + "name": "materializes __proto__, constructor, and prototype as ordinary own keys", + "input": "__proto__: polluted\nconstructor: 1\nprototype: 2", "expected": { + "__proto__": "polluted", "constructor": 1, "prototype": 2 }, - "specSection": "15" + "specSection": "15", + "note": "Decoding MUST NOT mutate the host object model, so the decoded object carries these keys as own entries" }, { "name": "materializes __proto__ opening a nested object as an ordinary own key", @@ -487,7 +400,7 @@ "specSection": "15" }, { - "name": "falls through to a key-value line when whitespace precedes the bracket segment", + "name": "falls through to a key-value line when whitespace precedes the bracket segment (non-strict)", "input": "foo [2]: bar,baz", "expected": { "foo [2]": "bar,baz" @@ -509,6 +422,123 @@ }, "specSection": "7.4", "minSpecVersion": "4.1" + }, + { + "name": "parses quoted empty string key opening a nested object", + "input": "\"\":\n a: 1", + "expected": { + "": { + "a": 1 + } + }, + "specSection": "8" + }, + { + "name": "parses quoted key shaped like an inline array header", + "input": "\"k[2]: y\": 1", + "expected": { + "k[2]: y": 1 + }, + "specSection": "5.2" + }, + { + "name": "materializes __proto__ array header key as an ordinary own key", + "input": "__proto__[2]: 1,2", + "expected": { + "__proto__": [ + 1, + 2 + ] + }, + "specSection": "15" + }, + { + "name": "materializes __proto__ entry key as an ordinary own key", + "input": "u[1:]{x}:\n __proto__: 1", + "expected": { + "u": { + "__proto__": { + "x": 1 + } + } + }, + "specSection": "15" + }, + { + "name": "parses an unquoted key followed by spaces before the colon", + "input": "a : 1", + "expected": { + "a": 1 + }, + "specSection": "7.4" + }, + { + "name": "parses a quoted key followed by spaces before the colon", + "input": "\"a\" : 1", + "expected": { + "a": 1 + }, + "specSection": "7.4", + "minSpecVersion": "4.1" + }, + { + "name": "parses a hyphen-leading line outside a list as a key-value line", + "input": "- a: 1", + "expected": { + "- a": 1 + }, + "specSection": "5.2" + }, + { + "name": "keeps keys that differ only in normalization form distinct", + "input": "\u00e9: 1\ne\u0301: 2", + "expected": { + "\u00e9": 1, + "e\u0301": 2 + }, + "specSection": "16", + "minSpecVersion": "4.1" + }, + { + "name": "parses an empty unquoted key as the empty key", + "input": ": 1", + "expected": { + "": 1 + }, + "specSection": "7.4", + "minSpecVersion": "4.2" + }, + { + "name": "parses a hyphen-leading line at list-item field depth as a key-value line", + "input": "items[1]:\n - a: 1\n - y: 2", + "expected": { + "items": [ + { + "a": 1, + "- y": 2 + } + ] + }, + "specSection": "5.2", + "minSpecVersion": "4.2" + }, + { + "name": "parses a key whose closed mid-token quote span hides a colon", + "input": "a\"b:c\"d: 1", + "expected": { + "a\"b:c\"d": 1 + }, + "specSection": "5.2", + "minSpecVersion": "4.2" + }, + { + "name": "reads a line whose only unquoted colon sits in the bracket segment as a key-value line", + "input": "a[2:]{x}", + "expected": { + "a[2": "]{x}" + }, + "specSection": "5.2", + "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/decode/primitives.json b/tests/fixtures/decode/primitives.json index 2347930..1be157c 100644 --- a/tests/fixtures/decode/primitives.json +++ b/tests/fixtures/decode/primitives.json @@ -9,34 +9,16 @@ "expected": "hello", "specSection": "7.4" }, - { - "name": "parses unquoted string with underscore and numbers", - "input": "Ada_99", - "expected": "Ada_99", - "specSection": "7.4" - }, { "name": "parses empty quoted string", "input": "\"\"", "expected": "", - "specSection": "7.4" - }, - { - "name": "parses quoted string with newline escape", - "input": "\"line1\\nline2\"", - "expected": "line1\nline2", - "specSection": "7.1" - }, - { - "name": "parses quoted string with tab escape", - "input": "\"tab\\there\"", - "expected": "tab\there", - "specSection": "7.1" + "specSection": "4" }, { - "name": "parses quoted string with carriage return escape", - "input": "\"return\\rcarriage\"", - "expected": "return\rcarriage", + "name": "parses quoted strings with newline, tab, and carriage return escapes", + "input": "[3]: \"line1\\nline2\",\"tab\\there\",\"return\\rcarriage\"", + "expected": ["line1\nline2", "tab\there", "return\rcarriage"], "specSection": "7.1" }, { @@ -65,27 +47,21 @@ }, { "name": "parses Unicode string", - "input": "café", - "expected": "café", - "specSection": "7.4" + "input": "caf\u00e9", + "expected": "caf\u00e9", + "specSection": "16" }, { "name": "parses Chinese characters", "input": "你好", "expected": "你好", - "specSection": "7.4" - }, - { - "name": "parses emoji", - "input": "🚀", - "expected": "🚀", - "specSection": "7.4" + "specSection": "16" }, { "name": "parses string with emoji and spaces", "input": "hello 👋 world", "expected": "hello 👋 world", - "specSection": "7.4" + "specSection": "16" }, { "name": "parses positive integer", @@ -127,49 +103,28 @@ "name": "respects ambiguity quoting for true", "input": "\"true\"", "expected": "true", - "specSection": "7.4" - }, - { - "name": "respects ambiguity quoting for false", - "input": "\"false\"", - "expected": "false", - "specSection": "7.4" - }, - { - "name": "respects ambiguity quoting for null", - "input": "\"null\"", - "expected": "null", - "specSection": "7.4" + "specSection": "4" }, { "name": "respects ambiguity quoting for integer", "input": "\"42\"", "expected": "42", - "specSection": "7.4" - }, - { - "name": "respects ambiguity quoting for negative decimal", - "input": "\"-3.14\"", - "expected": "-3.14", - "specSection": "7.4" - }, - { - "name": "respects ambiguity quoting for scientific notation", - "input": "\"1e-6\"", - "expected": "1e-6", - "specSection": "7.4" - }, - { - "name": "respects ambiguity quoting for leading-zero", - "input": "\"05\"", - "expected": "05", - "specSection": "7.4" + "specSection": "4" }, { "name": "decodes supplementary scalar (U+1F680) in quoted string as literal UTF-8", "input": "\"🚀 launch\"", "expected": "🚀 launch", "specSection": "7.1" + }, + { + "name": "decodes a raw control character inside a quoted string", + "input": "val: \"x\u0001y\"", + "expected": { + "val": "x\u0001y" + }, + "specSection": "7.1", + "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/decode/root-form.json b/tests/fixtures/decode/root-form.json index 32bc71f..081bdde 100644 --- a/tests/fixtures/decode/root-form.json +++ b/tests/fixtures/decode/root-form.json @@ -13,71 +13,102 @@ "specSection": "5" }, { - "name": "parses single primitive string at root as primitive", - "input": "hello", - "expected": "hello", + "name": "parses literal [] at root as empty array", + "input": "[]", + "expected": [], "options": { "strict": true }, "specSection": "5" }, { - "name": "parses single primitive number at root as primitive", - "input": "42", - "expected": 42, + "name": "throws on trailing content after a root array", + "input": "[2]: 1,2\njunk: 3", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "5", + "note": "The root form spans the whole document; trailing lines must not be silently discarded" + }, + { + "name": "throws on trailing content after a keyed tabular root", + "input": "[2:]{v}:\n a: 1\n b: 2\njunk: 3", + "expected": null, + "shouldError": true, "options": { "strict": true }, "specSection": "5" }, { - "name": "parses single primitive boolean at root as primitive", - "input": "true", - "expected": true, + "name": "throws on trailing content after a root empty array", + "input": "[]\njunk: 3", + "expected": null, + "shouldError": true, "options": { "strict": true }, "specSection": "5" }, { - "name": "parses literal [] at root as empty array", - "input": "[]", - "expected": [], + "name": "parses a root primitive after a leading blank line", + "input": "\nhello", + "expected": "hello", "options": { "strict": true }, "specSection": "5" }, { - "name": "throws on trailing content after a root array", - "input": "[2]: 1,2\njunk: 3", + "name": "throws on a trailing bare token after a root array in non-strict mode", + "input": "[2]: 1,2\njunk", "expected": null, "shouldError": true, "options": { - "strict": true + "strict": false }, "specSection": "5", - "note": "The root form spans the whole document; trailing lines must not be silently discarded" + "note": "The non-strict leniency for trailing content excludes scalar lines (§5.2)", + "minSpecVersion": "4.1" }, { - "name": "throws on trailing content after a keyed tabular root", - "input": "[2:]{v}:\n a: 1\n b: 2\njunk: 3", + "name": "parses a root line shaped like a header without a colon as a string", + "input": "items[2]", + "expected": "items[2]", + "specSection": "6", + "minSpecVersion": "4.2" + }, + { + "name": "throws on an indented root primitive in non-strict mode", + "input": " hello", "expected": null, "shouldError": true, "options": { - "strict": true + "strict": false }, - "specSection": "5" + "specSection": "5", + "note": "The non-strict skip for over-indented lines excludes scalar lines (§5.2)", + "minSpecVersion": "4.2" }, { - "name": "throws on trailing content after a root empty array", - "input": "[]\njunk: 3", + "name": "parses a root primitive whose colon follows an unclosed quote", + "input": "a\"b: 1", + "expected": "a\"b: 1", + "specSection": "5.2", + "minSpecVersion": "4.2" + }, + { + "name": "throws on a scalar line after a skipped over-indented first line in non-strict mode", + "input": " a: 1\nhello", "expected": null, "shouldError": true, "options": { - "strict": true + "strict": false }, - "specSection": "5" + "specSection": "5", + "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/decode/validation-errors.json b/tests/fixtures/decode/validation-errors.json index e3ad253..48055b5 100644 --- a/tests/fixtures/decode/validation-errors.json +++ b/tests/fixtures/decode/validation-errors.json @@ -36,7 +36,7 @@ "input": "\"a\\x\"", "expected": null, "shouldError": true, - "specSection": "14.2" + "specSection": "7.1" }, { "name": "throws on truncated unicode escape \\u00b", @@ -57,7 +57,7 @@ "input": "\"unterminated", "expected": null, "shouldError": true, - "specSection": "14.2" + "specSection": "7.1" }, { "name": "throws on missing colon in key-value context", @@ -81,7 +81,7 @@ "input": "items[2\t]{a\tb}:\n 1,2\n 3,4", "expected": null, "shouldError": true, - "specSection": "14.1", + "specSection": "11.2", "note": "Active delimiter is tab; rows using comma each parse as 1 value, failing the row width check" }, { @@ -136,11 +136,11 @@ "specSection": "14.3" }, { - "name": "throws on array header missing colon", + "name": "throws on a header-shaped line without a colon above indented rows", "input": "items[2]{id,name}\n 1,Ada\n 2,Bob", "expected": null, "shouldError": true, - "specSection": "6" + "specSection": "5.2" }, { "name": "throws when inline values fall short of the declared length", @@ -213,7 +213,7 @@ }, { "name": "throws on whitespace between bracket segment and colon in strict mode", - "input": "items[2] :\n 1,2", + "input": "items[2] : 1,2", "expected": null, "shouldError": true, "options": { @@ -224,7 +224,7 @@ }, { "name": "throws on whitespace between bracket segment and field list in strict mode", - "input": "items[2] {a,b}:\n 1,2\n 3,4", + "input": "items[0] {a,b}:", "expected": null, "shouldError": true, "options": { @@ -271,14 +271,14 @@ }, { "name": "throws on empty field list in strict mode", - "input": "items[1]{}:\n 1", + "input": "items[0]{}:", "expected": null, "shouldError": true, "specSection": "14.2" }, { "name": "throws on empty nested field group in strict mode", - "input": "items[1]{id,meta{}}:\n 1", + "input": "items[0]{id,meta{}}:", "expected": null, "shouldError": true, "specSection": "14.2", @@ -286,21 +286,20 @@ }, { "name": "throws on unmatched brace in field list in strict mode", - "input": "items[1]{id,customer{name:\n 1,Ada", + "input": "items[0]{id,customer{name:", "expected": null, "shouldError": true, "specSection": "14.2" }, { "name": "throws on duplicate field names at the same field-list level in strict mode", - "input": "items[1]{a,a{x}}:\n 1,2", + "input": "items[0]{a,a{x}}:", "expected": null, "shouldError": true, "options": { "strict": true }, - "specSection": "14.3", - "note": "A bare field and a nested field group sharing one name yield duplicate sibling keys in every decoded row" + "specSection": "9.3" }, { "name": "throws on entry row count mismatch with keyed header length", @@ -443,7 +442,7 @@ }, { "name": "throws on keyless fields-bearing header as list item", - "input": "items[1]:\n - [2]{x}:\n 1\n 2", + "input": "items[1]:\n - [0]{x}:", "expected": null, "shouldError": true, "specSection": "6" @@ -503,6 +502,232 @@ "specSection": "5.2", "note": "A scalar line outside root primitive position is an error in strict and non-strict mode alike", "minSpecVersion": "4.1" + }, + { + "name": "throws on the removed [#N] length marker in a root header in strict mode", + "input": "[#2]: a,b", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6" + }, + { + "name": "throws on the removed [#N] length marker in a field header in strict mode", + "input": "xs[#2]: a,b", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6" + }, + { + "name": "throws on the removed [#N] length marker in a keyless list-item header in strict mode", + "input": "items[1]:\n - [#2]: a,b", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6" + }, + { + "name": "throws on the removed [#N] length marker in a list-item first-field header in strict mode", + "input": "items[1]:\n - xs[#2]: a,b", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6" + }, + { + "name": "throws on tabular row count mismatch in a later field of a list-item object", + "input": "items[1]:\n - id: 1\n users[2]{id,name}:\n 1,Ada", + "expected": null, + "shouldError": true, + "specSection": "14.1" + }, + { + "name": "throws on a tab after a quoted value's closing quote", + "input": "k: \"a\"\t", + "expected": null, + "shouldError": true, + "specSection": "7.4", + "minSpecVersion": "4.1" + }, + { + "name": "throws on a tab after a quoted key's closing quote", + "input": "\"a\"\t: 1", + "expected": null, + "shouldError": true, + "specSection": "7.4", + "minSpecVersion": "4.1" + }, + { + "name": "throws on two primitives at root depth in non-strict mode", + "input": "hello\nworld", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "5", + "minSpecVersion": "4.1" + }, + { + "name": "throws on a surrogate pair escape", + "input": "val: \"\\ud83d\\ude00\"", + "expected": null, + "shouldError": true, + "specSection": "7.1" + }, + { + "name": "throws on a key-value line at row depth inside a tabular array", + "input": "items[2]{a,b}:\n 1,2\n c: 3,4", + "expected": null, + "shouldError": true, + "specSection": "9.3", + "note": "The colon precedes the delimiter, so the line ends the rows instead of becoming a second row" + }, + { + "name": "throws on a header delimiter mismatch that row widths do not expose", + "input": "items[1|]{a,b}:\n 1,2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "6" + }, + { + "name": "throws on a hyphen without a following space at item depth", + "input": "items[2]:\n - a\n -b", + "expected": null, + "shouldError": true, + "specSection": "5.2" + }, + { + "name": "throws on tabular rows at depth +1 under a list-item first-field header", + "input": "x[1]:\n - a[1]{p}:\n 1", + "expected": null, + "shouldError": true, + "specSection": "10" + }, + { + "name": "throws on invalid escape sequence in a quoted key", + "input": "\"a\\x\": 1", + "expected": null, + "shouldError": true, + "specSection": "7.4" + }, + { + "name": "throws on a [1] header with nothing after the colon", + "input": "a[1]:", + "expected": null, + "shouldError": true, + "specSection": "9.1", + "note": "Nothing after the colon is list form with zero items, not one empty inline value" + }, + { + "name": "throws on a length beyond the integer range that wraps to the value count", + "input": "a[18446744073709551617]: 1", + "expected": null, + "shouldError": true, + "specSection": "6", + "note": "2^64 + 1 wraps to 1 in 64-bit integers; an unrepresentable length is unmet (§14.1)", + "minSpecVersion": "4.2" + }, + { + "name": "throws on an empty field entry", + "input": "a[0]{x,}:", + "expected": null, + "shouldError": true, + "specSection": "6", + "minSpecVersion": "4.2" + }, + { + "name": "throws on a space between a field name and its nested field group", + "input": "a[0]{b {c}}:", + "expected": null, + "shouldError": true, + "specSection": "6", + "minSpecVersion": "4.2" + }, + { + "name": "throws on a nested field group without a name", + "input": "a[0]{x,{y}}:", + "expected": null, + "shouldError": true, + "specSection": "6", + "minSpecVersion": "4.2" + }, + { + "name": "throws on a \\u escape with a sign before its hex digits", + "input": "val: \"\\u+0ab\"", + "expected": null, + "shouldError": true, + "specSection": "7.1" + }, + { + "name": "throws on an escaped lone low surrogate", + "input": "val: \"\\udc00\"", + "expected": null, + "shouldError": true, + "specSection": "7.1" + }, + { + "name": "throws on characters after a closing quote in an inline array cell", + "input": "k[2]: \"a\"x,b", + "expected": null, + "shouldError": true, + "specSection": "7.4", + "minSpecVersion": "4.1" + }, + { + "name": "throws on a key-value line among tabular rows that breaks the declared count", + "input": "t[2]{a}:\n 1\n x: 2", + "expected": null, + "shouldError": true, + "specSection": "9.3" + }, + { + "name": "throws on a tabular row with more cells than fields", + "input": "items[1]{a}:\n 1,2", + "expected": null, + "shouldError": true, + "specSection": "14.1" + }, + { + "name": "throws on more keyed entry rows than declared", + "input": "m[1:]{v}:\n a: 1\n b: 2", + "expected": null, + "shouldError": true, + "specSection": "14.1" + }, + { + "name": "throws on an invalid escape in non-strict mode", + "input": "k: \"a\\x\"", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "7.1", + "minSpecVersion": "4.1" + }, + { + "name": "throws on an unterminated string in non-strict mode", + "input": "k: \"abc", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "7.1", + "minSpecVersion": "4.1" } ] } diff --git a/tests/fixtures/decode/whitespace.json b/tests/fixtures/decode/whitespace.json index fc22ed3..3585de6 100644 --- a/tests/fixtures/decode/whitespace.json +++ b/tests/fixtures/decode/whitespace.json @@ -4,27 +4,12 @@ "description": "Whitespace tolerance in decoding – surrounding spaces around delimiters and values", "tests": [ { - "name": "tolerates spaces around commas in inline arrays", - "input": "tags[3]: a , b , c", + "name": "tolerates spaces around comma, pipe, and tab delimiters in inline arrays", + "input": "comma[3]: a , b , c\npipe[3|]: a | b | c\ntab[3\t]: a \t b \t c", "expected": { - "tags": ["a", "b", "c"] - }, - "specSection": "12", - "note": "Trimming removes U+0020 only, never other Unicode whitespace" - }, - { - "name": "tolerates spaces around pipes in inline arrays", - "input": "tags[3|]: a | b | c", - "expected": { - "tags": ["a", "b", "c"] - }, - "specSection": "12" - }, - { - "name": "tolerates spaces around tabs in inline arrays", - "input": "tags[3\t]: a \t b \t c", - "expected": { - "tags": ["a", "b", "c"] + "comma": ["a", "b", "c"], + "pipe": ["a", "b", "c"], + "tab": ["a", "b", "c"] }, "specSection": "12" }, @@ -56,22 +41,23 @@ "specSection": "9.1" }, { - "name": "preserves NBSP-leading unquoted value", - "input": "k: \u00a0v", + "name": "keeps a leading NBSP and a trailing tab in unquoted values", + "input": "a: \u00a0v\nb: v\t", "expected": { - "k": "\u00a0v" + "a": "\u00a0v", + "b": "v\t" }, "specSection": "12", - "note": "Token trimming removes U+0020 only; NBSP is part of the token" + "note": "Token trimming removes U+0020 only; NBSP and tab are part of the token" }, { - "name": "preserves NBSP around inline array tokens", - "input": "items[2]: \u00a0a\u00a0,b", + "name": "preserves NBSP and tab around inline array tokens", + "input": "items[2]: \u00a0a\u00a0,\tb", "expected": { - "items": ["\u00a0a\u00a0", "b"] + "items": ["\u00a0a\u00a0", "\tb"] }, "specSection": "12", - "note": "A host trim() that strips the full Unicode whitespace set corrupts this round-trip" + "note": "Token trimming removes U+0020 only; a host trim() that also strips NBSP or tab corrupts this round-trip" }, { "name": "decodes CRLF line terminators", @@ -124,6 +110,33 @@ "specSection": "12", "note": "Trailing spaces are stripped before line classification", "minSpecVersion": "4.1" + }, + { + "name": "keeps a second carriage return before CRLF as content", + "input": "a: x\r\r\nb: 1", + "expected": { + "a": "x\r", + "b": 1 + }, + "specSection": "12" + }, + { + "name": "keeps a second leading byte-order mark as content", + "input": "\ufeff\ufeffa: 1", + "expected": { + "\ufeffa": 1 + }, + "specSection": "12", + "minSpecVersion": "4.1" + }, + { + "name": "removes a byte-order mark before a comment line", + "input": "\ufeff# c\na: 1", + "expected": { + "a": 1 + }, + "specSection": "12", + "minSpecVersion": "4.1" } ] } diff --git a/tests/fixtures/encode/arrays-nested.json b/tests/fixtures/encode/arrays-nested.json index 69c570e..079ddfb 100644 --- a/tests/fixtures/encode/arrays-nested.json +++ b/tests/fixtures/encode/arrays-nested.json @@ -3,14 +3,6 @@ "category": "encode", "description": "Nested and mixed array encoding – arrays of arrays, mixed type arrays, root arrays", "tests": [ - { - "name": "encodes nested arrays of primitives", - "input": { - "pairs": [["a", "b"], ["c", "d"]] - }, - "expected": "pairs[2]:\n - [2]: a,b\n - [2]: c,d", - "specSection": "9.2" - }, { "name": "quotes strings containing delimiters in nested arrays", "input": { @@ -27,14 +19,6 @@ "expected": "pairs[2]:\n - [0]:\n - [0]:", "specSection": "9.2" }, - { - "name": "encodes mixed-length inner arrays", - "input": { - "pairs": [[1], [2, 3]] - }, - "expected": "pairs[2]:\n - [1]: 1\n - [2]: 2,3", - "specSection": "9.2" - }, { "name": "encodes root-level primitive array", "input": ["x", "y", "true", true, 10], @@ -71,20 +55,6 @@ "expected": "[]", "specSection": "9.1" }, - { - "name": "encodes complex nested structure", - "input": { - "user": { - "id": 123, - "name": "Ada", - "tags": ["reading", "gaming"], - "active": true, - "prefs": [] - } - }, - "expected": "user:\n id: 123\n name: Ada\n tags[2]: reading,gaming\n active: true\n prefs: []", - "specSection": "8" - }, { "name": "uses list form for arrays mixing primitives and objects", "input": { @@ -111,22 +81,41 @@ }, { "name": "uses list form for a tabular-eligible array in list-item position", + "input": { + "a": [[{ "x": 1 }, { "x": 2 }]] + }, + "expected": "a[1]:\n - [2]:\n - x: 1\n - x: 2", + "specSection": "9.4", + "note": "A keyless fields-bearing header is valid only at the document root", + "minSpecVersion": "4.1" + }, + { + "name": "uses list form for an array of arrays in list-item position", + "input": [ + [ + [ + 1 + ] + ] + ], + "expected": "[1]:\n - [1]:\n - [1]: 1", + "specSection": "9.4" + }, + { + "name": "uses list form for an array mixing primitives and objects in list-item position", "input": { "a": [ [ + 1, { "x": 1 - }, - { - "x": 2 } - ] + ], + 2 ] }, - "expected": "a[1]:\n - [2]:\n - x: 1\n - x: 2", - "specSection": "9.3", - "note": "A keyless fields-bearing header is valid only at the document root", - "minSpecVersion": "4.1" + "expected": "a[2]:\n - [2]:\n - 1\n - x: 1\n - 2", + "specSection": "9.4" } ] } diff --git a/tests/fixtures/encode/arrays-objects.json b/tests/fixtures/encode/arrays-objects.json index ad8342b..ba3bf01 100644 --- a/tests/fixtures/encode/arrays-objects.json +++ b/tests/fixtures/encode/arrays-objects.json @@ -15,31 +15,14 @@ "specSection": "9.4" }, { - "name": "uses list form for objects with nested values", + "name": "preserves field order in list items", "input": { "items": [ - { "id": 1, "nested": { "x": 1 } }, - { "id": 2, "nested": { "y": 2 } } + { "nums": [1, 2, 3], "name": "Ada" }, + { "name": "Ada", "nums": [1, 2, 3] } ] }, - "expected": "items[2]:\n - id: 1\n nested:\n x: 1\n - id: 2\n nested:\n y: 2", - "specSection": "9.4", - "note": "The nested column mixes key sets {x} and {y}, so it is not nested-uniform (§9.3), so the array uses list form" - }, - { - "name": "preserves field order in list items - array first", - "input": { - "items": [{ "nums": [1, 2, 3], "name": "Ada" }] - }, - "expected": "items[1]:\n - nums[3]: 1,2,3\n name: Ada", - "specSection": "10" - }, - { - "name": "preserves field order in list items - primitive first", - "input": { - "items": [{ "name": "Ada", "nums": [1, 2, 3] }] - }, - "expected": "items[1]:\n - name: Ada\n nums[3]: 1,2,3", + "expected": "items[2]:\n - nums[3]: 1,2,3\n name: Ada\n - name: Ada\n nums[3]: 1,2,3", "specSection": "10" }, { @@ -53,13 +36,14 @@ "specSection": "10" }, { - "name": "uses tabular form for nested uniform object arrays", + "name": "uses tabular form for a list-item object's first field, with and without sibling fields", "input": { "items": [ - { "users": [{ "id": 1, "name": "Ada" }, { "id": 2, "name": "Bob" }], "status": "active" } + { "users": [{ "id": 1, "name": "Ada" }, { "id": 2, "name": "Bob" }], "status": "active" }, + { "users": [{ "id": 3, "name": "Cy" }, { "id": 4, "name": "Dee" }] } ] }, - "expected": "items[1]:\n - users[2]{id,name}:\n 1,Ada\n 2,Bob\n status: active", + "expected": "items[2]:\n - users[2]{id,name}:\n 1,Ada\n 2,Bob\n status: active\n - users[2]{id,name}:\n 3,Cy\n 4,Dee", "specSection": "10", "note": "Tabular header on hyphen line with rows at depth +2 and sibling fields at depth +1" }, @@ -74,66 +58,16 @@ "specSection": "10" }, { - "name": "uses list form for objects with multiple array fields", - "input": { - "items": [{ "nums": [1, 2], "tags": ["a", "b"], "name": "test" }] - }, - "expected": "items[1]:\n - nums[2]: 1,2\n tags[2]: a,b\n name: test", - "specSection": "10" - }, - { - "name": "uses list form for objects with only array fields", - "input": { - "items": [{ "nums": [1, 2, 3], "tags": ["a", "b"] }] - }, - "expected": "items[1]:\n - nums[3]: 1,2,3\n tags[2]: a,b", - "specSection": "10" - }, - { - "name": "encodes objects with empty arrays in list form", + "name": "places empty arrays on the hyphen line when first and at field depth when later", "input": { "items": [ - { "name": "Ada", "data": [] } + { "data": [], "name": "Ada" }, + { "name": "Bob", "data": [] } ] }, - "expected": "items[1]:\n - name: Ada\n data: []", + "expected": "items[2]:\n - data: []\n name: Ada\n - name: Bob\n data: []", "specSection": "10" }, - { - "name": "uses canonical encoding for multi-field list-item objects with tabular arrays", - "input": { - "items": [{ "users": [{ "id": 1 }, { "id": 2 }], "note": "x" }] - }, - "expected": "items[1]:\n - users[2]{id}:\n 1\n 2\n note: x", - "specSection": "10", - "note": "Tabular header on hyphen line with rows at depth +2 and sibling fields at depth +1" - }, - { - "name": "uses canonical encoding for single-field list-item tabular arrays", - "input": { - "items": [{ "users": [{ "id": 1, "name": "Ada" }, { "id": 2, "name": "Bob" }] }] - }, - "expected": "items[1]:\n - users[2]{id,name}:\n 1,Ada\n 2,Bob", - "specSection": "10", - "note": "Tabular header on hyphen line with rows at depth +2" - }, - { - "name": "places empty arrays on hyphen line when first", - "input": { - "items": [{ "data": [], "name": "x" }] - }, - "expected": "items[1]:\n - data: []\n name: x", - "specSection": "10" - }, - { - "name": "encodes empty object list items as bare hyphen", - "input": { - "items": ["first", "second", {}] - }, - "expected": "items[3]:\n - first\n - second\n -", - "specSection": "10", - "note": "Empty object list items encode as a single \"-\" line at the list-item depth" - }, { "name": "uses field order from first object for tabular headers", "input": { @@ -186,6 +120,41 @@ "specSection": "9.5", "note": "Keyed tabular form is mandated in object-field and root positions, not in a column", "minSpecVersion": "4.1" + }, + { + "name": "places an empty object first field on the hyphen line", + "input": { + "items": [ + { + "a": {}, + "b": 1 + } + ] + }, + "expected": "items[1]:\n - a:\n b: 1", + "specSection": "10" + }, + { + "name": "uses tabular form for a later field of a list-item object", + "input": { + "items": [ + { + "id": 1, + "users": [ + { + "id": 1, + "name": "Ada" + }, + { + "id": 2, + "name": "Bob" + } + ] + } + ] + }, + "expected": "items[1]:\n - id: 1\n users[2]{id,name}:\n 1,Ada\n 2,Bob", + "specSection": "10" } ] } diff --git a/tests/fixtures/encode/arrays-primitive.json b/tests/fixtures/encode/arrays-primitive.json index 5644091..5a2ab0e 100644 --- a/tests/fixtures/encode/arrays-primitive.json +++ b/tests/fixtures/encode/arrays-primitive.json @@ -3,22 +3,6 @@ "category": "encode", "description": "Primitive array encoding – inline arrays of strings, numbers, booleans", "tests": [ - { - "name": "encodes string arrays inline", - "input": { - "tags": ["reading", "gaming"] - }, - "expected": "tags[2]: reading,gaming", - "specSection": "9.1" - }, - { - "name": "encodes number arrays inline", - "input": { - "nums": [1, 2, 3] - }, - "expected": "nums[3]: 1,2,3", - "specSection": "9.1" - }, { "name": "encodes mixed primitive arrays inline", "input": { @@ -84,7 +68,7 @@ "specSection": "9.1" }, { - "name": "quotes strings that look like booleans in arrays", + "name": "quotes strings that look like literals or numbers in arrays", "input": { "items": ["x", "true", "42", "-3.14"] }, diff --git a/tests/fixtures/encode/arrays-tabular.json b/tests/fixtures/encode/arrays-tabular.json index 2365042..f013733 100644 --- a/tests/fixtures/encode/arrays-tabular.json +++ b/tests/fixtures/encode/arrays-tabular.json @@ -83,7 +83,7 @@ "input": { "orders": [ { "id": 1, "customer": { "name": "Ada", "country": "DK" }, "total": 99 }, - { "id": 2, "customer": { "name": "Bob", "country": "UK" }, "total": 149 } + { "id": 2, "customer": { "country": "UK", "name": "Bob" }, "total": 149 } ] }, "expected": "orders[2]{id,customer{name,country},total}:\n 1,Ada,DK,99\n 2,Bob,UK,149", diff --git a/tests/fixtures/encode/delimiters.json b/tests/fixtures/encode/delimiters.json index 03c9271..b8fbc61 100644 --- a/tests/fixtures/encode/delimiters.json +++ b/tests/fixtures/encode/delimiters.json @@ -122,17 +122,6 @@ }, "specSection": "11" }, - { - "name": "quotes strings containing tab delimiter", - "input": { - "items": ["a", "b\tc", "d"] - }, - "expected": "items[3\t]: a\t\"b\\tc\"\td", - "options": { - "delimiter": "\t" - }, - "specSection": "11" - }, { "name": "quotes strings containing pipe delimiter", "input": { @@ -155,31 +144,6 @@ }, "specSection": "11" }, - { - "name": "does not quote commas with pipe delimiter", - "input": { - "items": ["a,b", "c,d"] - }, - "expected": "items[2|]: a,b|c,d", - "options": { - "delimiter": "|" - }, - "specSection": "11" - }, - { - "name": "quotes tabular values containing comma delimiter", - "input": { - "items": [ - { "id": 1, "note": "a,b" }, - { "id": 2, "note": "c,d" } - ] - }, - "expected": "items[2]{id,note}:\n 1,\"a,b\"\n 2,\"c,d\"", - "options": { - "delimiter": "," - }, - "specSection": "11" - }, { "name": "does not quote commas in tabular values with tab delimiter", "input": { @@ -205,17 +169,6 @@ }, "specSection": "11" }, - { - "name": "does not quote commas in object values with tab delimiter", - "input": { - "note": "a,b" - }, - "expected": "note: a,b", - "options": { - "delimiter": "\t" - }, - "specSection": "11" - }, { "name": "quotes nested array values containing pipe delimiter", "input": { @@ -228,26 +181,22 @@ "specSection": "11" }, { - "name": "quotes nested array values containing tab delimiter", - "input": { - "pairs": [["a", "b\tc"]] - }, - "expected": "pairs[1\t]:\n - [2\t]: a\t\"b\\tc\"", - "options": { - "delimiter": "\t" - }, - "specSection": "11" - }, - { - "name": "preserves ambiguity quoting regardless of delimiter", + "name": "quotes a primitive list item containing the document delimiter", "input": { - "items": ["true", "42", "-3.14"] + "a": [ + "x|y", + "p,q", + { + "b": 1 + } + ] }, - "expected": "items[3|]: \"true\"|\"42\"|\"-3.14\"", + "expected": "a[3|]:\n - \"x|y\"\n - p,q\n - b: 1", "options": { "delimiter": "|" }, - "specSection": "11" + "specSection": "11.1", + "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/encode/objects-keyed.json b/tests/fixtures/encode/objects-keyed.json index 7fd4e87..d8841f5 100644 --- a/tests/fixtures/encode/objects-keyed.json +++ b/tests/fixtures/encode/objects-keyed.json @@ -62,14 +62,10 @@ { "name": "quotes entry keys per key encoding", "input": { - "ids": { - "42": { "v": 1 }, - "my key": { "v": 2 } - } + "ids": { "42": { "v": 1 }, "my key": { "v": 2 } } }, "expected": "ids[2:]{v}:\n \"42\": 1\n \"my key\": 2", - "specSection": "9.5", - "note": "Entry keys follow §7.3, the same rule as object keys" + "specSection": "9.5" }, { "name": "quotes entry-row cells containing the active delimiter", @@ -150,6 +146,14 @@ "expected": "items[2]:\n - a:\n x: 1\n b:\n x: 2\n - 5", "specSection": "9.5", "note": "The element object is keyed-eligible but has no key; the keyless form is root-only, so it encodes per §10" + }, + { + "name": "keeps an object of empty objects out of keyed tabular form", + "input": { + "m": { "a": {}, "b": {} } + }, + "expected": "m:\n a:\n b:", + "specSection": "9.5" } ] } diff --git a/tests/fixtures/encode/objects.json b/tests/fixtures/encode/objects.json index a63f92c..7b5b145 100644 --- a/tests/fixtures/encode/objects.json +++ b/tests/fixtures/encode/objects.json @@ -4,26 +4,18 @@ "description": "Object encoding – simple objects, nested objects, key encoding", "tests": [ { - "name": "preserves key order in objects", + "name": "preserves key order in objects with primitive values", "input": { "id": 123, "name": "Ada", - "active": true - }, - "expected": "id: 123\nname: Ada\nactive: true", - "specSection": "8" - }, - { - "name": "encodes null values in objects", - "input": { - "id": 123, + "active": true, "value": null }, - "expected": "id: 123\nvalue: null", + "expected": "id: 123\nname: Ada\nactive: true\nvalue: null", "specSection": "8" }, { - "name": "encodes empty objects as empty string", + "name": "encodes an empty root object as an empty document", "input": {}, "expected": "", "specSection": "8" @@ -93,11 +85,11 @@ "specSection": "7.2" }, { - "name": "quotes string value that looks like negative decimal", + "name": "quotes string value that looks like a number with an uppercase exponent", "input": { - "v": "-7.5" + "v": "1E6" }, - "expected": "v: \"-7.5\"", + "expected": "v: \"1E6\"", "specSection": "7.2" }, { @@ -109,19 +101,12 @@ "specSection": "7.3" }, { - "name": "quotes key with brackets", - "input": { - "[index]": 5 - }, - "expected": "\"[index]\": 5", - "specSection": "7.3" - }, - { - "name": "quotes key with braces", + "name": "quotes keys with brackets or braces", "input": { - "{key}": 5 + "[index]": 5, + "{key}": 6 }, - "expected": "\"{key}\": 5", + "expected": "\"[index]\": 5\n\"{key}\": 6", "specSection": "7.3" }, { @@ -133,11 +118,12 @@ "specSection": "7.3" }, { - "name": "quotes key with spaces", + "name": "quotes keys with spaces", "input": { - "full name": "Ada" + "full name": "Ada", + " a ": 1 }, - "expected": "\"full name\": Ada", + "expected": "\"full name\": Ada\n\" a \": 1", "specSection": "7.3" }, { @@ -148,14 +134,6 @@ "expected": "\"-lead\": 1", "specSection": "7.3" }, - { - "name": "quotes key with leading and trailing spaces", - "input": { - " a ": 1 - }, - "expected": "\" a \": 1", - "specSection": "7.3" - }, { "name": "quotes numeric key", "input": { @@ -182,19 +160,21 @@ "note": "The unquoted-key pattern is ASCII-only, so every non-ASCII key is quoted" }, { - "name": "quotes CJK key", + "name": "keeps a dotted key unquoted", "input": { - "名前": "x" + "a.b": 1 }, - "expected": "\"名前\": x", - "specSection": "7.3" + "expected": "a.b: 1", + "specSection": "7.3", + "minSpecVersion": "4.2" }, { - "name": "escapes newline in key", + "name": "escapes newline and quotes in keys", "input": { - "line\nbreak": 1 + "line\nbreak": 1, + "he said \"hi\"": 2 }, - "expected": "\"line\\nbreak\": 1", + "expected": "\"line\\nbreak\": 1\n\"he said \\\"hi\\\"\": 2", "specSection": "7.1" }, { @@ -205,14 +185,6 @@ "expected": "\"tab\\there\": 2", "specSection": "7.1" }, - { - "name": "escapes quotes in key", - "input": { - "he said \"hi\"": 1 - }, - "expected": "\"he said \\\"hi\\\"\": 1", - "specSection": "7.1" - }, { "name": "escapes U+0004 control character in key via \\uXXXX", "input": { @@ -259,15 +231,6 @@ "specSection": "15", "note": "No key has special meaning, so the encoder emits prototype-named own entries like any other (§8)" }, - { - "name": "encodes constructor and prototype own properties as ordinary keys", - "input": { - "constructor": 1, - "prototype": 2 - }, - "expected": "constructor: 1\nprototype: 2", - "specSection": "15" - }, { "name": "encodes __proto__ as a tabular field name", "input": { @@ -292,6 +255,27 @@ }, "expected": "note: \"#x\"", "specSection": "7.2" + }, + { + "name": "encodes empty string key opening a nested object", + "input": { + "": { + "a": 1 + } + }, + "expected": "\"\":\n a: 1", + "specSection": "8" + }, + { + "name": "keeps strings that only resemble numbers or literals unquoted", + "input": { + "a": ".5", + "b": "NaN", + "c": "True" + }, + "expected": "a: .5\nb: NaN\nc: True", + "specSection": "7.2", + "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/encode/primitives.json b/tests/fixtures/encode/primitives.json index 826c95e..d53756d 100644 --- a/tests/fixtures/encode/primitives.json +++ b/tests/fixtures/encode/primitives.json @@ -46,9 +46,9 @@ "specSection": "7.2" }, { - "name": "quotes string that looks like negative decimal", - "input": "-3.14", - "expected": "\"-3.14\"", + "name": "quotes string that looks like a decimal", + "input": "3.14", + "expected": "\"3.14\"", "specSection": "7.2" }, { @@ -65,21 +65,9 @@ "note": "Matches the numeric-like pattern, so it MUST be quoted" }, { - "name": "escapes newline in string", - "input": "line1\nline2", - "expected": "\"line1\\nline2\"", - "specSection": "7.1" - }, - { - "name": "escapes tab in string", - "input": "tab\there", - "expected": "\"tab\\there\"", - "specSection": "7.1" - }, - { - "name": "escapes carriage return in string", - "input": "return\rcarriage", - "expected": "\"return\\rcarriage\"", + "name": "escapes newline, tab, and carriage return in strings", + "input": ["line1\nline2", "tab\there", "return\rcarriage"], + "expected": "[3]: \"line1\\nline2\",\"tab\\there\",\"return\\rcarriage\"", "specSection": "7.1" }, { @@ -95,10 +83,11 @@ "specSection": "7.1" }, { - "name": "quotes string with array-like syntax", - "input": "[3]: x,y", - "expected": "\"[3]: x,y\"", - "specSection": "7.2" + "name": "keeps a string with an interior hash unquoted", + "input": "a #b", + "expected": "a #b", + "specSection": "7.2", + "minSpecVersion": "4.2" }, { "name": "quotes string starting with hyphen-space", @@ -107,28 +96,16 @@ "specSection": "7.2" }, { - "name": "quotes single hyphen as object value", - "input": { "marker": "-" }, - "expected": "marker: \"-\"", + "name": "quotes single hyphen and hyphen-space string as object values", + "input": { "marker": "-", "note": "- item" }, + "expected": "marker: \"-\"\nnote: \"- item\"", "specSection": "7.2", - "note": "Single hyphen must be quoted to avoid list item ambiguity" - }, - { - "name": "quotes string starting with hyphen as object value", - "input": { "note": "- item" }, - "expected": "note: \"- item\"", - "specSection": "7.2" + "note": "Either would otherwise read as a list item marker" }, { - "name": "quotes single hyphen in array", - "input": { "items": ["-"] }, - "expected": "items[1]: \"-\"", - "specSection": "7.2" - }, - { - "name": "quotes leading-hyphen string in array", - "input": { "tags": ["a", "- item", "b"] }, - "expected": "tags[3]: a,\"- item\",b", + "name": "quotes single hyphen and leading-hyphen string in array", + "input": { "items": ["-", "-x"] }, + "expected": "items[2]: \"-\",\"-x\"", "specSection": "7.2" }, { @@ -149,18 +126,6 @@ "expected": "café", "specSection": "7.2" }, - { - "name": "encodes Chinese characters without quotes", - "input": "你好", - "expected": "你好", - "specSection": "7.2" - }, - { - "name": "encodes emoji without quotes", - "input": "🚀", - "expected": "🚀", - "specSection": "7.2" - }, { "name": "encodes string with emoji and spaces", "input": "hello 👋 world", @@ -200,13 +165,13 @@ }, { "name": "encodes negative zero as zero", - "input": -0, + "input": -0.0, "expected": "0", "specSection": "2" }, { "name": "encodes large integer without exponent notation", - "input": 1000000, + "input": 1e6, "expected": "1000000", "specSection": "2", "note": "Within the canonical decimal range" @@ -231,9 +196,9 @@ "specSection": "2" }, { - "name": "encodes repeating decimal with full precision", - "input": 0.3333333333333333, - "expected": "0.3333333333333333", + "name": "encodes floats with their shortest round-trip digits", + "input": [0.3333333333333333, 0.30000000000000004], + "expected": "[2]: 0.3333333333333333,0.30000000000000004", "specSection": "2" }, { @@ -281,6 +246,14 @@ "specSection": "7.2", "note": "Unquoted, a decoder would remove the U+FEFF as a byte-order mark (§12) and read the number 8", "minSpecVersion": "4.1" + }, + { + "name": "encodes a large integer-valued number with its shortest round-trip digits", + "input": 123456789012345680000, + "expected": "123456789012345680000", + "specSection": "2", + "note": "The binary64 value is exactly 123456789012345683968", + "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/encode/whitespace.json b/tests/fixtures/encode/whitespace.json index 1c6eae3..07dda0a 100644 --- a/tests/fixtures/encode/whitespace.json +++ b/tests/fixtures/encode/whitespace.json @@ -3,14 +3,6 @@ "category": "encode", "description": "Whitespace and formatting invariants – no trailing spaces, no trailing newlines", "tests": [ - { - "name": "produces no trailing newline at end of output", - "input": { - "id": 123 - }, - "expected": "id: 123", - "specSection": "12" - }, { "name": "maintains proper indentation for nested structures", "input": { @@ -46,6 +38,17 @@ "specSection": "7.2", "note": "The whitespace trigger is U+0020 and U+0009 only", "minSpecVersion": "4.1" + }, + { + "name": "indents list-item sibling fields by indentSize", + "input": { + "a": [{ "b": 1, "c": 2 }, 1] + }, + "expected": "a[2]:\n - b: 1\n c: 2\n - 1", + "options": { + "indentSize": 4 + }, + "specSection": "10" } ] } From 5d51cb389b615367634c1a6caa32e5b22a0bb0f8 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:10:06 +0200 Subject: [PATCH 02/15] fix(decode): read an indented first line as over-indented --- src/toon_format/_decoder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 3a890e7..1887c7a 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -459,7 +459,8 @@ def field_list( # -- document (§5) ------------------------------------------------------ def document(self) -> Any: - first = self.peek() + while (first := self.peek()) is not None and first.depth > 0: + self.orphan(first) # an indented first line is over-indented (§5, §8) if first is None: return self.finish({}) # empty document (§5) if first.content == "[]": @@ -467,7 +468,7 @@ def document(self) -> Any: self.end_of_root() return [] kind = self.classify(first.content, first) - if isinstance(kind, _Header) and kind.key is None and first.depth == 0: + if isinstance(kind, _Header) and kind.key is None: self.take() value = self.header_value(kind, first, 1) self.end_of_root() From f888659d0d6ee7460a80303b9da0485227208c87 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:10:23 +0200 Subject: [PATCH 03/15] fix(decode): reject a scalar line after a root array in non-strict mode --- src/toon_format/_decoder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 1887c7a..d4f23a1 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -484,6 +484,9 @@ def end_of_root(self) -> None: if line is not None: if self.strict: raise self.error("Unexpected content after the root value", line) + for trailing in self.lines[self.pos :]: + if isinstance(self.classify(trailing.content, trailing), _Scalar): + raise self.error("Unexpected bare value", trailing) self.pos = len(self.lines) # -- objects (§8) ------------------------------------------------------- From 3519e75fc88909bbac6375d947d918845a01e962 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:10:41 +0200 Subject: [PATCH 04/15] fix(decode): skip over-indented lines inside array scopes in non-strict mode --- src/toon_format/_decoder.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index d4f23a1..8eb0ea4 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -569,11 +569,12 @@ def items(self, header: _Header, line: _Line, depth: int) -> list[Any]: started = False self.lists += 1 try: - while ( - (item := self.peek()) - and item.depth == depth - and _is_list_item(item.content) - ): + while (item := self.peek()) and item.depth >= depth: + if item.depth > depth: + self.orphan(item) + continue + if not _is_list_item(item.content): + break self.take() if not started: self.spans += 1 @@ -619,7 +620,10 @@ def rows(self, header: _Header, line: _Line, depth: int) -> list[Any]: rows: list[Any] = [] started = False try: - while (row := self.peek()) and row.depth == depth: + while (row := self.peek()) and row.depth >= depth: + if row.depth > depth: + self.orphan(row) + continue colon = _find_unquoted(row.content, ":") if colon >= 0: split = _find_unquoted(row.content, delimiter) @@ -643,7 +647,10 @@ def entries(self, header: _Header, line: _Line, depth: int) -> Any: obj: dict[str, Any] = {} count = 0 try: - while (entry := self.peek()) and entry.depth == depth: + while (entry := self.peek()) and entry.depth >= depth: + if entry.depth > depth: + self.orphan(entry) + continue self.take() if not count: self.spans += 1 From d1ecb4e7ae6a1ff7c8725caa3fe86a73d1d06f89 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:11:27 +0200 Subject: [PATCH 05/15] fix(decode): adopt the depth of a jumped first line in non-strict mode --- src/toon_format/_decoder.py | 40 +++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 8eb0ea4..91fbc4d 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -476,7 +476,7 @@ def document(self) -> Any: if isinstance(kind, _Scalar) and len(self.lines) == 1: self.take() return self.primitive(first.content, first) - return self.finish(self.object_body(0, {})) + return self.finish(self.object_body(0, {}, -1)) def end_of_root(self) -> None: """A root array or keyed object spans the whole document (§5).""" @@ -491,12 +491,14 @@ def end_of_root(self) -> None: # -- objects (§8) ------------------------------------------------------- - def object_body(self, depth: int, obj: dict[str, Any]) -> dict[str, Any]: - """Parse the fields of an object whose content sits at ``depth``.""" - while (line := self.peek()) is not None and line.depth >= depth: + def object_body( + self, depth: int, obj: dict[str, Any], parent: int + ) -> dict[str, Any]: + """Parse the fields at ``depth`` of an object opened at ``parent`` (§8).""" + while (line := self.peek()) is not None and line.depth > parent: if self.lists and _is_list_item(line.content): raise self.error("List item outside the item depth of its array", line) - if line.depth > depth: + if line.depth != depth: self.orphan(line) continue self.take() @@ -538,21 +540,21 @@ def field_value(self, rest: str, line: _Line, depth: int) -> Any: return self.finish({}) if self.strict and child.depth != depth + 1: raise self.error("Indentation jumps more than one level", child) - return self.finish(self.object_body(child.depth, {})) + return self.finish(self.object_body(child.depth, {}, depth)) # -- arrays and keyed objects (§9, §10) --------------------------------- def header_value(self, header: _Header, line: _Line, depth: int) -> Any: """Decode the value a header opens; ``depth`` is its content depth.""" if header.keyed: - return self.entries(header, line, self.scope_depth(depth)) + return self.entries(header, line, self.scope_depth(depth), depth - 1) if header.fields is not None: - return self.rows(header, line, self.scope_depth(depth)) + return self.rows(header, line, self.scope_depth(depth), depth - 1) if header.rest: values = self.values(header.rest, header.delimiter, line) self.check_count(len(values), header, line) return values - return self.items(header, line, self.scope_depth(depth)) + return self.items(header, line, self.scope_depth(depth), depth - 1) def check_count(self, count: int, header: _Header, line: _Line) -> None: if self.strict and count != header.length: @@ -563,14 +565,14 @@ def check_width(self, cells: list[Any], fields: Fields, line: _Line) -> None: if self.strict and len(cells) != width: raise self.error(f"Expected {width} values but found {len(cells)}", line) - def items(self, header: _Header, line: _Line, depth: int) -> list[Any]: + def items(self, header: _Header, line: _Line, depth: int, parent: int) -> list[Any]: """Parse the list items of an array in list form (§9.2, §9.4).""" items: list[Any] = [] started = False self.lists += 1 try: - while (item := self.peek()) and item.depth >= depth: - if item.depth > depth: + while (item := self.peek()) and item.depth > parent: + if item.depth != depth: self.orphan(item) continue if not _is_list_item(item.content): @@ -611,17 +613,17 @@ def item(self, line: _Line, depth: int) -> Any: obj[kind.key] = self.header_value(kind, line, depth + 2) else: obj[kind.key] = self.field_value(kind.rest, line, depth + 1) - return self.finish(self.object_body(depth + 1, obj)) + return self.finish(self.object_body(depth + 1, obj, depth)) - def rows(self, header: _Header, line: _Line, depth: int) -> list[Any]: + def rows(self, header: _Header, line: _Line, depth: int, parent: int) -> list[Any]: """Parse the rows of a tabular array (§9.3).""" assert header.fields is not None delimiter = header.delimiter rows: list[Any] = [] started = False try: - while (row := self.peek()) and row.depth >= depth: - if row.depth > depth: + while (row := self.peek()) and row.depth > parent: + if row.depth != depth: self.orphan(row) continue colon = _find_unquoted(row.content, ":") @@ -641,14 +643,14 @@ def rows(self, header: _Header, line: _Line, depth: int) -> list[Any]: self.check_count(len(rows), header, line) return rows - def entries(self, header: _Header, line: _Line, depth: int) -> Any: + def entries(self, header: _Header, line: _Line, depth: int, parent: int) -> Any: """Parse the entry rows of a keyed tabular object (§9.5).""" assert header.fields is not None obj: dict[str, Any] = {} count = 0 try: - while (entry := self.peek()) and entry.depth >= depth: - if entry.depth > depth: + while (entry := self.peek()) and entry.depth > parent: + if entry.depth != depth: self.orphan(entry) continue self.take() From ee9b9ac2ce574995dc600c95f81e2e2e0c83dd8a Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:12:01 +0200 Subject: [PATCH 06/15] fix(decode): read a hyphen line at list-item field depth as a key-value line --- src/toon_format/_decoder.py | 7 ------- tests/test_decode.py | 19 ------------------- 2 files changed, 26 deletions(-) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 91fbc4d..b8bcefa 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -228,9 +228,6 @@ def __init__( # Number of enclosing header scopes that have consumed their first # item, row, or entry: a blank line inside any of them is a header span. self.spans = 0 - # Number of enclosing arrays in list form. Inside them a line starting - # with "- " is a list item, admissible only at item depth (§5.2). - self.lists = 0 # -- infrastructure ----------------------------------------------------- @@ -496,8 +493,6 @@ def object_body( ) -> dict[str, Any]: """Parse the fields at ``depth`` of an object opened at ``parent`` (§8).""" while (line := self.peek()) is not None and line.depth > parent: - if self.lists and _is_list_item(line.content): - raise self.error("List item outside the item depth of its array", line) if line.depth != depth: self.orphan(line) continue @@ -569,7 +564,6 @@ def items(self, header: _Header, line: _Line, depth: int, parent: int) -> list[A """Parse the list items of an array in list form (§9.2, §9.4).""" items: list[Any] = [] started = False - self.lists += 1 try: while (item := self.peek()) and item.depth > parent: if item.depth != depth: @@ -583,7 +577,6 @@ def items(self, header: _Header, line: _Line, depth: int, parent: int) -> list[A started = True items.append(self.item(item, depth)) finally: - self.lists -= 1 self.spans -= started self.check_count(len(items), header, line) return items diff --git a/tests/test_decode.py b/tests/test_decode.py index 05e5c15..428ddfe 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -266,25 +266,6 @@ def test_hyphen_outside_a_list_is_part_of_the_key() -> None: } -@pytest.mark.parametrize( - "text", - [ - # A list item where a field of the list-item object is expected (§10). - "items[1]:\n - a: 1\n - b: 2", - "items[1]:\n - a:\n - b: 2", - # A list item too deep to be an item of the array. - "items[2]:\n - a: 1\n - b: 2", - "items[1]:\n - [1]:\n - x\n - y", - ], -) -def test_list_item_outside_item_depth(text: str) -> None: - # §5.2: inside an array in list form a leading hyphen always marks a list - # item, which is admissible only at the array's item depth. - for strict in (True, False): - with pytest.raises(toon_format.ToonDecodeError): - toon_format.loads(text, strict=strict) - - def test_integer_beyond_the_conversion_limit() -> None: if not hasattr(sys, "set_int_max_str_digits"): pytest.skip("no integer string conversion limit") From aa688ae765d78cd466e177d9cda306b863ff0b66 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:12:24 +0200 Subject: [PATCH 07/15] fix(decode): read a line whose only colon is in the bracket segment as a key-value line --- src/toon_format/_decoder.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index b8bcefa..7bb02d3 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -174,6 +174,27 @@ def _find_unquoted(text: str, char: str, start: int = 0) -> int: return -1 +def _brace_end(text: str, start: int) -> int: + """Index of the ``}`` closing the ``{`` at ``start``, outside quotes, or -1.""" + depth = 0 + i, n = start, len(text) + while i < n: + c = text[i] + if c == '"': + i = _quoted_end(text, i) + if i < 0: + return -1 + continue + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + return i + i += 1 + return -1 + + def _split(text: str, delimiter: str) -> list[str]: """Split on unquoted delimiters and trim each token (§11.2, Appendix B.3).""" if '"' not in text: @@ -393,7 +414,12 @@ def header( except _Malformed as exc: if colon < 0: return _Scalar(content) # without a colon it is a scalar line (§5.2) - if self.strict: + # A line whose only colons sit in the bracket segment or its field + # list is a key-value line, in strict mode too (§5.2, §6). + close = _find_unquoted(content, "]", bracket) + if close >= 0 and content.startswith("{", close + 1): + close = max(close, _brace_end(content, close + 1)) + if self.strict and close >= 0 and _find_unquoted(content, ":", close) >= 0: raise self.error(str(exc), line) from None return self.literal_key_value(content, line) key_text = content[:bracket] @@ -401,7 +427,7 @@ def header( return _Header(key, int(match.group(1)), keyed, delimiter, fields, rest) def literal_key_value(self, content: str, line: _Line) -> _KeyValue: - """Read a malformed header as a key-value line (non-strict fall-through, §6).""" + """Read a rejected header line as a key-value line (§5.2, §6).""" colon = _find_unquoted(content, ":") return _KeyValue( self.key(content[:colon], line), content[colon + 1 :].strip(" ") From 4dc1a5fc1f1ba9877e3a06a12dad0b4ea1561430 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:12:36 +0200 Subject: [PATCH 08/15] fix(decode): ignore braces and delimiters in a mid-name quote span of a field list --- src/toon_format/_decoder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 7bb02d3..74a3580 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -453,7 +453,11 @@ def field_list( else: start = pos while pos < n and text[pos] not in "{},|\t": - pos += 1 + if text[pos] == '"': + end = _quoted_end(text, pos) + pos = end if end >= 0 else n + else: + pos += 1 name = text[start:pos].strip(" ") if not name: raise _Malformed("Empty field name in field list") From 3deec753da22ed9fd6fe8ad22e21c25dcac3061e Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:12:52 +0200 Subject: [PATCH 09/15] fix(decode): reject whitespace before a nested field group --- src/toon_format/_decoder.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 74a3580..be487f2 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -465,6 +465,10 @@ def field_list( pos += 1 nested = None if text.startswith("{", pos): + if text[pos - 1] == " ": + raise _Malformed( + "Whitespace between a field name and its nested field group" + ) nested, pos = self.field_list(text, pos, delimiter, line) if name in seen and self.strict: raise self.error(f"Duplicate field name {name!r}", line) From c67b806204487104bf252d0ada45dccc108e794d Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:13:16 +0200 Subject: [PATCH 10/15] fix(decode): accept raw control characters inside quoted strings in strict mode --- docs/data-types.md | 3 +-- src/toon_format/_decoder.py | 3 --- tests/test_decode.py | 9 --------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/docs/data-types.md b/docs/data-types.md index 82593a5..e40e786 100644 --- a/docs/data-types.md +++ b/docs/data-types.md @@ -45,8 +45,7 @@ default strict decoder: pass `parse_float=Decimal` to read it. **Strings.** A string holding an unpaired surrogate cannot be encoded and raises `ValueError`. A string starting with U+FEFF is quoted, because a -leading U+FEFF in a document is read as a byte-order mark. In strict mode, quoted strings may not contain literal -control characters other than tab. +leading U+FEFF in a document is read as a byte-order mark. **Key order** is preserved, except that tabular rows follow the header's field order. With `strict=False`, a duplicate key keeps its first position and its diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index be487f2..4e24142 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -28,7 +28,6 @@ _BRACKET = re.compile(r"\[(0|[1-9][0-9]*)(:?)([\t|]?)\]") _QUOTED = re.compile(r'"(?:[^"\\]|\\.)*"', re.DOTALL) _HEX4 = re.compile(r"[0-9A-Fa-f]{4}") -_CONTROL = re.compile(r"[\x00-\x08\x0a-\x1f]") _SIMPLE_ESCAPES = {"\\": "\\", '"': '"', "n": "\n", "r": "\r", "t": "\t"} _MISSING = object() @@ -292,8 +291,6 @@ def scope_depth(self, expected: int) -> int: # -- tokens (§4, §7) ---------------------------------------------------- def unescape(self, body: str, line: _Line) -> str: - if self.strict and _CONTROL.search(body): - raise self.error("Unescaped control character in quoted string", line) if "\\" not in body: return body parts: list[str] = [] diff --git a/tests/test_decode.py b/tests/test_decode.py index 428ddfe..28ae11d 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -166,15 +166,6 @@ def test_errors_in_any_mode(text: str) -> None: toon_format.loads(text, strict=strict) -def test_literal_control_characters_in_quotes() -> None: - with pytest.raises(toon_format.ToonDecodeError, match="control character"): - toon_format.loads('a: "x\x01y"') - assert toon_format.loads('a: "x\x01y"', strict=False) == {"a": "x\x01y"} - assert toon_format.loads('a: "x\ty"') == { - "a": "x\ty" - } # a literal tab is tolerated (§7.1) - - def test_deep_nesting_raises_decode_error() -> None: text = "\n".join(" " * depth + "a:" for depth in range(5000)) with pytest.raises(toon_format.ToonDecodeError, match="nested too deeply"): From 3410ec6d924d893b8b5dd88a5223ad8646ec25d6 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:13:35 +0200 Subject: [PATCH 11/15] fix(decode): treat a tab-only line as indentation in strict mode --- src/toon_format/_decoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 4e24142..67232a6 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -83,7 +83,8 @@ def _split_lines(text: str, indent_size: int, strict: bool) -> list[_Line]: if stripped.startswith("#"): continue # comment line (§5.1); lines are adjacent across it content = stripped.rstrip(" ") - if not content.strip("\t"): + # Tabs make a line blank only where they may indent it (§12). + if not content or (not strict and not content.strip(" \t")): blank = blank or number continue spaces = len(source) - len(stripped) From 0b42cdd4b6c2f652aceb9bab1dd5b0a91ba06d3f Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:14:57 +0200 Subject: [PATCH 12/15] feat: target TOON spec 4.2 --- AGENTS.md | 2 +- CHANGELOG.md | 16 ++++++++++++++++ README.md | 6 +++--- docs/data-types.md | 2 +- docs/index.md | 2 +- src/toon_format/__init__.py | 2 +- tests/test_api.py | 2 +- 7 files changed, 24 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 574405c..8928e41 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ and mirrors the `json` module: `dumps`, `dump`, `loads`, `load`, and `ToonDecodeError`, with `encode` and `decode` as aliases of `dumps` and `loads`, the names the other TOON implementations use. -**Target specification: TOON 4.1**, exposed as `toon_format.__toon_spec__`. +**Target specification: TOON 4.2**, exposed as `toon_format.__toon_spec__`. The conformance fixtures in `tests/fixtures/` are copied from the spec repository and never edited by hand. The specification is the authority: when this code, its tests, or another implementation disagree with diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ac2408..60e10e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,22 @@ All notable changes to this project are documented here. The format follows ## [Unreleased] +### Changed + +- Support for [TOON specification 4.2](https://github.com/toon-format/spec/blob/v4.2.1/SPEC.md): + all official fixtures of spec `v4.2.1` pass, and `toon_format.__toon_spec__` + is `"4.2"`. Encoder output is unchanged. +- Decoding settles the edge cases that 4.1 left open: quoted strings accept raw + control characters in strict mode, a leading hyphen marks a list item only at + item depth, a line whose only colons sit in the bracket segment or its field + list is a key-value line, a quote span inside a field name hides braces and + delimiters, whitespace before a nested field group is a header error, a + tab-only line is a tab-indentation error in strict mode instead of a blank + line, and an indented first line is over-indented. +- With `strict=False`, a jumped first line sets the depth of its scope, + over-indented lines inside arrays are skipped instead of ending them, and a + scalar line after a root array is an error. + ## [1.0.0] A complete rewrite. See [docs/migration.md](docs/migration.md) for upgrading diff --git a/README.md b/README.md index c2eff29..430aa86 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,14 @@ [![CI](https://github.com/toon-format/toon-python/actions/workflows/ci.yml/badge.svg)](https://github.com/toon-format/toon-python/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/toon-format.svg)](https://pypi.org/project/toon-format/) [![Python versions](https://img.shields.io/pypi/pyversions/toon-format.svg)](https://pypi.org/project/toon-format/) -[![TOON spec](https://img.shields.io/badge/TOON%20spec-4.1-blue)](https://github.com/toon-format/spec/blob/main/SPEC.md) +[![TOON spec](https://img.shields.io/badge/TOON%20spec-4.2-blue)](https://github.com/toon-format/spec/blob/main/SPEC.md) [![Documentation](https://readthedocs.org/projects/toon-format/badge/?version=stable)](https://toon-format.readthedocs.io/en/stable/) The official Python implementation of **TOON** (Token-Oriented Object Notation): a compact, human-readable encoding of the JSON data model, designed to save tokens in LLM prompts. -- **Complete**: implements [TOON specification 4.1](https://github.com/toon-format/spec/blob/main/SPEC.md) +- **Complete**: implements [TOON specification 4.2](https://github.com/toon-format/spec/blob/main/SPEC.md) and passes every official conformance fixture. - **Familiar**: `dumps`, `dump`, `loads`, and `load`, like the `json` module, with `encode` and `decode` as aliases. @@ -119,7 +119,7 @@ implementations, and Pydantic. The distribution and the module keep their names, and `encode(value)` and `decode(text)` keep working. Options are now keyword arguments instead of a dictionary, the token helpers and the `toon` command are gone, and the output -follows specification 4.1. See the +follows specification 4.2. See the [migration guide](https://toon-format.readthedocs.io/en/stable/migration/). ## Contributing diff --git a/docs/data-types.md b/docs/data-types.md index e40e786..19661b8 100644 --- a/docs/data-types.md +++ b/docs/data-types.md @@ -30,7 +30,7 @@ converted back to dates or other types. ## Specification choices -This package targets **TOON 4.1** (`toon_format.__toon_spec__`). The specification +This package targets **TOON 4.2** (`toon_format.__toon_spec__`). The specification asks implementations to document the following choices. **Numbers.** Integers encode and decode exactly at any size. Floats use the diff --git a/docs/index.md b/docs/index.md index 7c9f85d..37beed1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ The official Python implementation of **TOON** (Token-Oriented Object Notation): a compact, human-readable encoding of the JSON data model, designed to save tokens in LLM prompts. -- **Complete**: implements [TOON specification 4.1](https://github.com/toon-format/spec/blob/main/SPEC.md) +- **Complete**: implements [TOON specification 4.2](https://github.com/toon-format/spec/blob/main/SPEC.md) and passes every official conformance fixture. - **Familiar**: `dumps`, `dump`, `loads`, and `load`, like the `json` module, with `encode` and `decode` as aliases. diff --git a/src/toon_format/__init__.py b/src/toon_format/__init__.py index e993388..959305a 100644 --- a/src/toon_format/__init__.py +++ b/src/toon_format/__init__.py @@ -37,7 +37,7 @@ ToonDecodeError.__module__ = __name__ # shown as toon_format.ToonDecodeError -__toon_spec__ = "4.1" +__toon_spec__ = "4.2" """Version of the TOON specification this package implements.""" try: diff --git a/tests/test_api.py b/tests/test_api.py index 292de78..9b914b0 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -27,7 +27,7 @@ def test_public_names() -> None: "loads", ] ) - assert toon_format.__toon_spec__ == "4.1" + assert toon_format.__toon_spec__ == "4.2" assert re.fullmatch(r"\d+\.\d+\.\d+.*", toon_format.__version__) From d917b0a852394d56a8767812bab1bcbdd7a960c3 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:28:50 +0200 Subject: [PATCH 13/15] fix(encode): quote a leading U+FEFF only in a root string --- CHANGELOG.md | 3 ++- docs/data-types.md | 2 +- src/toon_format/_encoder.py | 4 +++- src/toon_format/_text.py | 2 +- tests/test_encode.py | 6 +++--- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e10e5..57bd03d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ All notable changes to this project are documented here. The format follows - Support for [TOON specification 4.2](https://github.com/toon-format/spec/blob/v4.2.1/SPEC.md): all official fixtures of spec `v4.2.1` pass, and `toon_format.__toon_spec__` - is `"4.2"`. Encoder output is unchanged. + is `"4.2"`. A string starting with U+FEFF is quoted only as a root + primitive. - Decoding settles the edge cases that 4.1 left open: quoted strings accept raw control characters in strict mode, a leading hyphen marks a list item only at item depth, a line whose only colons sit in the bracket segment or its field diff --git a/docs/data-types.md b/docs/data-types.md index 19661b8..8cebf32 100644 --- a/docs/data-types.md +++ b/docs/data-types.md @@ -44,7 +44,7 @@ keeps them when decoding. A `Decimal` beyond the double range, such as default strict decoder: pass `parse_float=Decimal` to read it. **Strings.** A string holding an unpaired surrogate cannot be encoded and -raises `ValueError`. A string starting with U+FEFF is quoted, because a +raises `ValueError`. A root string starting with U+FEFF is quoted, because a leading U+FEFF in a document is read as a byte-order mark. **Key order** is preserved, except that tabular rows follow the header's field diff --git a/src/toon_format/_encoder.py b/src/toon_format/_encoder.py index 7e50da0..98e5eb4 100644 --- a/src/toon_format/_encoder.py +++ b/src/toon_format/_encoder.py @@ -19,7 +19,7 @@ from typing import Any, TypeAlias from uuid import UUID -from ._text import encode_key, encode_string, format_decimal, format_float +from ._text import encode_key, encode_string, format_decimal, format_float, quote __all__ = ["encode"] @@ -268,6 +268,8 @@ def root(self, value: Json) -> None: self.array("", value, 0, tabular=True) else: self.emit(0, "[]") + elif isinstance(value, str) and value.startswith("\ufeff"): + self.emit(0, quote(value)) # unquoted, it would read as a BOM (§7.2, §12) else: self.emit(0, self.primitive(value)) diff --git a/src/toon_format/_text.py b/src/toon_format/_text.py index f7c34d1..ba96fcd 100644 --- a/src/toon_format/_text.py +++ b/src/toon_format/_text.py @@ -67,7 +67,7 @@ def quote(value: str) -> str: def _needs_quotes(value: str, delimiter: str) -> bool: return ( not value - or value[0] in " \t-#\ufeff" # a leading U+FEFF reads as a BOM (§12) + or value[0] in " \t-#" or value[-1] in " \t" or value in ("true", "false", "null") or delimiter in value diff --git a/tests/test_encode.py b/tests/test_encode.py index 4bbeea8..eaca672 100644 --- a/tests/test_encode.py +++ b/tests/test_encode.py @@ -413,10 +413,10 @@ def test_non_finite_float_keys(key: float, expected: str) -> None: ("value", "expected"), [ ("\ufeffabc", '"\ufeffabc"'), - ({"a": "\ufeffx"}, 'a: "\ufeffx"'), - (["\ufeffx"], '[1]: "\ufeffx"'), + ({"a": "\ufeffx"}, "a: \ufeffx"), + (["\ufeffx"], "[1]: \ufeffx"), ], ) -def test_leading_byte_order_mark_is_quoted(value: Any, expected: str) -> None: +def test_leading_byte_order_mark(value: Any, expected: str) -> None: assert toon_format.dumps(value) == expected assert toon_format.loads(expected) == value From 513810ad42d7e71b0bdec3a8d6cf18abce574216 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:30:09 +0200 Subject: [PATCH 14/15] fix(decode): trim spaces after a nested field group --- src/toon_format/_decoder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 67232a6..99d5a6f 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -468,6 +468,8 @@ def field_list( "Whitespace between a field name and its nested field group" ) nested, pos = self.field_list(text, pos, delimiter, line) + while pos < n and text[pos] == " ": + pos += 1 if name in seen and self.strict: raise self.error(f"Duplicate field name {name!r}", line) seen.add(name) From 983b4479312b75327bb93de1266a8f67b5d7b049 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 10:31:21 +0200 Subject: [PATCH 15/15] test: align the hyphen key test with spec 4.2 --- tests/test_decode.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_decode.py b/tests/test_decode.py index 28ae11d..3d675a5 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -247,9 +247,8 @@ def test_row_or_key_value_at_row_depth() -> None: def test_hyphen_outside_a_list_is_part_of_the_key() -> None: - # §5.2: outside the scope of an array in list form a leading hyphen is - # ordinary text. - assert toon_format.loads("- a: 1") == {"- a": 1} + # §5.2: a leading hyphen marks a list item only at item depth; elsewhere it + # is ordinary text. assert toon_format.loads("- a") == "- a" assert toon_format.loads("items[1]:\n - x\n- a: 1", strict=False) == { "items": ["x"],