From 72fdad0c5877d3139bc614c28c5c0e5b07479931 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 16:20:16 +0200 Subject: [PATCH 01/13] feat: target TOON spec 4.3 --- AGENTS.md | 2 +- CHANGELOG.md | 10 ++ README.md | 6 +- docs/data-types.md | 2 +- docs/index.md | 2 +- src/toon_format/__init__.py | 2 +- src/toon_format/_decoder.py | 28 +---- tests/fixtures/README.md | 2 +- tests/fixtures/decode/arrays-tabular.json | 7 +- tests/fixtures/decode/blank-lines.json | 37 ++++--- tests/fixtures/decode/delimiters.json | 7 +- tests/fixtures/decode/indentation-errors.json | 27 ++++- tests/fixtures/decode/objects.json | 48 +++++--- tests/fixtures/decode/root-form.json | 29 ++++- tests/fixtures/decode/validation-errors.json | 104 +++++++++++++++--- tests/fixtures/encode/delimiters.json | 16 ++- tests/fixtures/encode/objects-keyed.json | 7 +- tests/fixtures/encode/objects.json | 16 ++- tests/fixtures/encode/primitives.json | 13 ++- tests/test_api.py | 2 +- 20 files changed, 257 insertions(+), 110 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8928e41..5fdc0ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ and mirrors the `json` module: `dumps`, `dump`, `loads`, `load`, and `ToonDecodeError`, with `encode` and `decode` as aliases of `dumps` and `loads`, the names the other TOON implementations use. -**Target specification: TOON 4.2**, exposed as `toon_format.__toon_spec__`. +**Target specification: TOON 4.3**, exposed as `toon_format.__toon_spec__`. The conformance fixtures in `tests/fixtures/` are copied from the spec repository and never edited by hand. The specification is the authority: when this code, its tests, or another implementation disagree with diff --git a/CHANGELOG.md b/CHANGELOG.md index e113102..7d4daa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ All notable changes to this project are documented here. The format follows ## [Unreleased] +### Changed + +- Support for [TOON specification 4.3](https://github.com/toon-format/spec/blob/v4.3.0/SPEC.md): + all official fixtures of spec `v4.3.0` pass, and `toon_format.__toon_spec__` + is `"4.3"`. Encoder output is unchanged. +- In strict mode, a line whose first unquoted `[` precedes its first unquoted + colon and that fails the header grammar is an error, also when its only + colon sits in the bracket segment or field list (`a[1:`, `a[2:]{x}`). With + `strict=False` it is still a key-value line. + ## [1.1.0] - 2026-10-06 ### Changed diff --git a/README.md b/README.md index 5b01ef0..b9b97b2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![CI](https://github.com/toon-format/toon-python/actions/workflows/ci.yml/badge.svg)](https://github.com/toon-format/toon-python/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/toon-format.svg)](https://pypi.org/project/toon-format/) [![Python versions](https://img.shields.io/pypi/pyversions/toon-format.svg)](https://pypi.org/project/toon-format/) -[![TOON spec](https://img.shields.io/badge/TOON%20spec-4.2-blue)](https://github.com/toon-format/spec/blob/main/SPEC.md) +[![TOON spec](https://img.shields.io/badge/TOON%20spec-4.3-blue)](https://github.com/toon-format/spec/blob/main/SPEC.md) [![Documentation](https://readthedocs.org/projects/toon-format/badge/?version=stable)](https://toon-format.readthedocs.io/en/stable/) The official Python implementation of **TOON** (Token-Oriented Object Notation): @@ -11,7 +11,7 @@ a compact, human-readable encoding of the JSON data model, designed to save tokens in LLM prompts. Keep working with dictionaries and lists in your code, and encode them as TOON only where a model reads them. -- **Complete**: implements [TOON specification 4.2](https://github.com/toon-format/spec/blob/main/SPEC.md) +- **Complete**: implements [TOON specification 4.3](https://github.com/toon-format/spec/blob/main/SPEC.md) and passes every official conformance fixture. - **Familiar**: `dumps`, `dump`, `loads`, and `load`, like the `json` module, plus [`encode` and `decode`](#encode-and-decode), the names of the @@ -174,7 +174,7 @@ implementations, and Pydantic. The distribution and the module keep their names, and `encode(value)` and `decode(text)` keep working. Options are now keyword arguments instead of a dictionary, the token helpers and the `toon` command are gone, and the output -follows specification 4.2. See the +follows specification 4.3. See the [migration guide](https://toon-format.readthedocs.io/en/stable/migration/). ## Contributing diff --git a/docs/data-types.md b/docs/data-types.md index 8cebf32..5f143c0 100644 --- a/docs/data-types.md +++ b/docs/data-types.md @@ -30,7 +30,7 @@ converted back to dates or other types. ## Specification choices -This package targets **TOON 4.2** (`toon_format.__toon_spec__`). The specification +This package targets **TOON 4.3** (`toon_format.__toon_spec__`). The specification asks implementations to document the following choices. **Numbers.** Integers encode and decode exactly at any size. Floats use the diff --git a/docs/index.md b/docs/index.md index 37beed1..b4a247f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ The official Python implementation of **TOON** (Token-Oriented Object Notation): a compact, human-readable encoding of the JSON data model, designed to save tokens in LLM prompts. -- **Complete**: implements [TOON specification 4.2](https://github.com/toon-format/spec/blob/main/SPEC.md) +- **Complete**: implements [TOON specification 4.3](https://github.com/toon-format/spec/blob/main/SPEC.md) and passes every official conformance fixture. - **Familiar**: `dumps`, `dump`, `loads`, and `load`, like the `json` module, with `encode` and `decode` as aliases. diff --git a/src/toon_format/__init__.py b/src/toon_format/__init__.py index 959305a..880065a 100644 --- a/src/toon_format/__init__.py +++ b/src/toon_format/__init__.py @@ -37,7 +37,7 @@ ToonDecodeError.__module__ = __name__ # shown as toon_format.ToonDecodeError -__toon_spec__ = "4.2" +__toon_spec__ = "4.3" """Version of the TOON specification this package implements.""" try: diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 99d5a6f..c1f8fe2 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -174,27 +174,6 @@ def _find_unquoted(text: str, char: str, start: int = 0) -> int: return -1 -def _brace_end(text: str, start: int) -> int: - """Index of the ``}`` closing the ``{`` at ``start``, outside quotes, or -1.""" - depth = 0 - i, n = start, len(text) - while i < n: - c = text[i] - if c == '"': - i = _quoted_end(text, i) - if i < 0: - return -1 - continue - if c == "{": - depth += 1 - elif c == "}": - depth -= 1 - if depth == 0: - return i - i += 1 - return -1 - - def _split(text: str, delimiter: str) -> list[str]: """Split on unquoted delimiters and trim each token (§11.2, Appendix B.3).""" if '"' not in text: @@ -412,12 +391,7 @@ def header( except _Malformed as exc: if colon < 0: return _Scalar(content) # without a colon it is a scalar line (§5.2) - # A line whose only colons sit in the bracket segment or its field - # list is a key-value line, in strict mode too (§5.2, §6). - close = _find_unquoted(content, "]", bracket) - if close >= 0 and content.startswith("{", close + 1): - close = max(close, _brace_end(content, close + 1)) - if self.strict and close >= 0 and _find_unquoted(content, ":", close) >= 0: + if self.strict: raise self.error(str(exc), line) from None return self.literal_key_value(content, line) key_text = content[:bracket] diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index fd50375..510ee2e 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -2,5 +2,5 @@ The `encode/` and `decode/` directories are copied verbatim from the [TOON specification repository](https://github.com/toon-format/spec/tree/main/tests/fixtures) -at tag `v4.2.1`. Do not edit them by hand: run `uv run scripts/update_fixtures.py ` +at tag `v4.3.0`. Do not edit them by hand: run `uv run scripts/update_fixtures.py ` to replace them with another release. diff --git a/tests/fixtures/decode/arrays-tabular.json b/tests/fixtures/decode/arrays-tabular.json index 1f01652..1292e1e 100644 --- a/tests/fixtures/decode/arrays-tabular.json +++ b/tests/fixtures/decode/arrays-tabular.json @@ -209,12 +209,15 @@ }, { "name": "trims spaces around field names", - "input": "items[1]{ a , b }:\n 1,2", + "input": "items[1]{ a , n{x} , b }:\n 1,2,3", "expected": { "items": [ { "a": 1, - "b": 2 + "n": { + "x": 2 + }, + "b": 3 } ] }, diff --git a/tests/fixtures/decode/blank-lines.json b/tests/fixtures/decode/blank-lines.json index a25a5a9..cbeaefe 100644 --- a/tests/fixtures/decode/blank-lines.json +++ b/tests/fixtures/decode/blank-lines.json @@ -33,16 +33,6 @@ }, "specSection": "14.2" }, - { - "name": "throws on multiple blank lines inside array", - "input": "items[2]:\n - a\n\n\n - b", - "expected": null, - "shouldError": true, - "options": { - "strict": true - }, - "specSection": "14.2" - }, { "name": "throws on blank line with spaces inside array", "input": "items[2]:\n - a\n \n - b", @@ -89,9 +79,10 @@ }, { "name": "accepts trailing newline at end of file", - "input": "a: 1\n", + "input": "a: 1\nitems[1]:\n - b: 2\n", "expected": { - "a": 1 + "a": 1, + "items": [{ "b": 2 }] }, "options": { "strict": true @@ -110,11 +101,15 @@ "specSection": "12" }, { - "name": "accepts blank line after array ends", - "input": "items[1]:\n - a\n\nb: 2", + "name": "accepts blank line after array ends, at root and in a nested object", + "input": "items[1]:\n - a\n\nb: 2\nc:\n d[1]:\n - x\n\n e: 3", "expected": { "items": ["a"], - "b": 2 + "b": 2, + "c": { + "d": ["x"], + "e": 3 + } }, "options": { "strict": true @@ -137,9 +132,9 @@ }, { "name": "ignores blank lines inside list array when strict=false", - "input": "items[3]:\n - a\n\n - b\n\n\n - c", + "input": "items[4]:\n - a\n\n - b\n\n\n - c\n - x: 1\n\n y: 2", "expected": { - "items": ["a", "b", "c"] + "items": ["a", "b", "c", { "x": 1, "y": 2 }] }, "options": { "strict": false @@ -210,6 +205,14 @@ "specSection": "14.2", "note": "The blank is after the nested object's content but inside the outer array's span (§12)" }, + { + "name": "throws on blank line between a list-item field and its nested object", + "input": "a[1]:\n - k:\n\n x: 1", + "expected": null, + "shouldError": true, + "specSection": "14.2", + "note": "The blank is between the field and its nested object's first line but inside the array's header span (§12)" + }, { "name": "throws on a tab-only line, which is not blank", "input": "a: 1\n\t\nb: 2", diff --git a/tests/fixtures/decode/delimiters.json b/tests/fixtures/decode/delimiters.json index 45b01d7..26a7272 100644 --- a/tests/fixtures/decode/delimiters.json +++ b/tests/fixtures/decode/delimiters.json @@ -198,12 +198,15 @@ "minSpecVersion": "4.1" }, { - "name": "treats a delimiter inside a mid-token quote span as quoted", - "input": "k[2]: a\"b,c\",d", + "name": "treats a delimiter inside a closed or unclosed mid-token quote span as quoted", + "input": "k[2]: a\"b,c\",d\nu[1]: x\"y,z", "expected": { "k": [ "a\"b,c\"", "d" + ], + "u": [ + "x\"y,z" ] }, "specSection": "5.2", diff --git a/tests/fixtures/decode/indentation-errors.json b/tests/fixtures/decode/indentation-errors.json index 378f080..16e4b9e 100644 --- a/tests/fixtures/decode/indentation-errors.json +++ b/tests/fixtures/decode/indentation-errors.json @@ -206,12 +206,16 @@ }, { "name": "adopts the depth of a jumped first line in non-strict mode", - "input": "a:\n b: 1\n c: 2\n d: 3", + "input": "a:\n b: 1\n c: 2\n d: 3\nitems[2]:\n - 1\n - 2", "expected": { "a": { "b": 1, "d": 3 - } + }, + "items": [ + 1, + 2 + ] }, "options": { "strict": false @@ -255,13 +259,14 @@ "minSpecVersion": "4.2" }, { - "name": "skips an over-indented line under a primitive list item in non-strict mode", - "input": "items[2]:\n - a\n b: 1\n - c", + "name": "skips over-indented lines under a primitive list item and after a list scope in non-strict mode", + "input": "items[2]:\n - a\n b: 1\n - c\n x: 1\nz: 2", "expected": { "items": [ "a", "c" - ] + ], + "z": 2 }, "options": { "strict": false @@ -269,6 +274,18 @@ "specSection": "8", "minSpecVersion": "4.2" }, + { + "name": "throws on an over-indented scalar line under a primitive list item in non-strict mode", + "input": "items[2]:\n - a\n b\n - c", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "8", + "note": "The non-strict skip for over-indented lines excludes scalar lines (§5.2)", + "minSpecVersion": "4.1" + }, { "name": "skips an over-indented line under a list-item field in non-strict mode", "input": "items[1]:\n - a: 1\n deep: 1\n b: 2", diff --git a/tests/fixtures/decode/objects.json b/tests/fixtures/decode/objects.json index bca97f8..5cf1b8a 100644 --- a/tests/fixtures/decode/objects.json +++ b/tests/fixtures/decode/objects.json @@ -200,16 +200,18 @@ "note": "An absent length prevents header interpretation; non-strict fall-through produces a literal key" }, { - "name": "treats non-integer bracket content as literal key (non-strict)", - "input": "foo[bar][1]: 20", + "name": "treats bracket content that fails the length grammar as literal key (non-strict)", + "input": "foo[bar][1]: 20\nn[-1]: 1\nk[\u0662]: a,b", "options": { "strict": false }, "expected": { - "foo[bar][1]": 20 + "foo[bar][1]": 20, + "n[-1]": 1, + "k[\u0662]": "a,b" }, "specSection": "6", - "note": "[bar] fails integer parsing; non-strict fall-through produces a literal key not constrained by §7.3" + "note": "[bar], [-1] and [\u0662] (a non-ASCII digit) fail the length grammar; non-strict fall-through produces a literal key not constrained by §7.3" }, { "name": "parses quoted key with comma", @@ -344,10 +346,12 @@ "note": "Encoders must quote hyphen-leading strings (§7.2); decoders accept the unquoted form as data" }, { - "name": "accepts unquoted colon-containing value in strict mode", - "input": "k: b:c", + "name": "accepts unquoted values and keys an encoder would quote in strict mode", + "input": "k: b:c\npaths[2]: C:\\,D:\\\na\\: b", "expected": { - "k": "b:c" + "k": "b:c", + "paths": ["C:\\", "D:\\"], + "a\\": "b" }, "specSection": "7.4", "note": "The first unquoted colon ends the key; the entire post-colon token is the value (§11.2)" @@ -501,42 +505,56 @@ }, { "name": "parses an empty unquoted key as the empty key", - "input": ": 1", + "input": ": 1\na:\n : 2\nitems[1]:\n - : 3\nm[1:]{v}:\n : 4", "expected": { - "": 1 + "": 1, + "a": { "": 2 }, + "items": [{ "": 3 }], + "m": { "": { "v": 4 } } }, "specSection": "7.4", "minSpecVersion": "4.2" }, { - "name": "parses a hyphen-leading line at list-item field depth as a key-value line", - "input": "items[1]:\n - a: 1\n - y: 2", + "name": "parses a hyphen-leading line off item depth as a key-value line or a tabular row", + "input": "items[1]:\n - a: 1\n - y: 2\nrows[1]{a,b}:\n - y,2", "expected": { "items": [ { "a": 1, "- y": 2 } + ], + "rows": [ + { + "a": "- y", + "b": 2 + } ] }, "specSection": "5.2", "minSpecVersion": "4.2" }, { - "name": "parses a key whose closed mid-token quote span hides a colon", - "input": "a\"b:c\"d: 1", + "name": "parses keys whose closed mid-token quote span hides a colon or a bracket segment, or precedes one", + "input": "a\"b:c\"d: 1\ne\"[1]:\"f: 2\na\"b\"[2]: x,y", "expected": { - "a\"b:c\"d": 1 + "a\"b:c\"d": 1, + "e\"[1]:\"f": 2, + "a\"b\"": ["x", "y"] }, "specSection": "5.2", "minSpecVersion": "4.2" }, { - "name": "reads a line whose only unquoted colon sits in the bracket segment as a key-value line", + "name": "falls through to a key-value line in non-strict mode when a header line fails the grammar", "input": "a[2:]{x}", "expected": { "a[2": "]{x}" }, + "options": { + "strict": false + }, "specSection": "5.2", "minSpecVersion": "4.2" } diff --git a/tests/fixtures/decode/root-form.json b/tests/fixtures/decode/root-form.json index 081bdde..10abbab 100644 --- a/tests/fixtures/decode/root-form.json +++ b/tests/fixtures/decode/root-form.json @@ -73,6 +73,17 @@ "note": "The non-strict leniency for trailing content excludes scalar lines (§5.2)", "minSpecVersion": "4.1" }, + { + "name": "throws on a bare token line under a trailing header after a root array in non-strict mode", + "input": "[1]: 1\nx[1]:\n junk", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "5", + "minSpecVersion": "4.1" + }, { "name": "parses a root line shaped like a header without a colon as a string", "input": "items[2]", @@ -93,9 +104,21 @@ "minSpecVersion": "4.2" }, { - "name": "parses a root primitive whose colon follows an unclosed quote", - "input": "a\"b: 1", - "expected": "a\"b: 1", + "name": "throws on an indented root array header", + "input": " [2]: 1,2", + "expected": null, + "shouldError": true, + "options": { + "strict": true + }, + "specSection": "5", + "note": "Root form discovery reads the first non-blank depth-0 line; an indented first line is over-indented (§8)", + "minSpecVersion": "4.2" + }, + { + "name": "parses a root primitive whose bracket segment and colon follow an unclosed quote", + "input": "k\"x[2]: 1,2", + "expected": "k\"x[2]: 1,2", "specSection": "5.2", "minSpecVersion": "4.2" }, diff --git a/tests/fixtures/decode/validation-errors.json b/tests/fixtures/decode/validation-errors.json index 48055b5..56a9a62 100644 --- a/tests/fixtures/decode/validation-errors.json +++ b/tests/fixtures/decode/validation-errors.json @@ -84,16 +84,6 @@ "specSection": "11.2", "note": "Active delimiter is tab; rows using comma each parse as 1 value, failing the row width check" }, - { - "name": "throws on mismatched delimiter between bracket and brace fields", - "input": "items[2\t]{a,b}:\n 1\t2\n 3\t4", - "expected": null, - "shouldError": true, - "options": { - "strict": true - }, - "specSection": "6" - }, { "name": "throws on extra brackets between bracket segment and colon in strict mode", "input": "foo[1][bar]: 10", @@ -142,6 +132,22 @@ "shouldError": true, "specSection": "5.2" }, + { + "name": "throws on a header line with an unclosed bracket segment", + "input": "a[1:", + "expected": null, + "shouldError": true, + "specSection": "5.2", + "minSpecVersion": "4.3" + }, + { + "name": "throws on a keyed header line without its colon", + "input": "a[2:]{x}", + "expected": null, + "shouldError": true, + "specSection": "5.2", + "minSpecVersion": "4.3" + }, { "name": "throws when inline values fall short of the declared length", "input": "tags[3]: a,b", @@ -156,6 +162,13 @@ "shouldError": true, "specSection": "14.1" }, + { + "name": "throws on a value line under a bare list-form header", + "input": "tags[3]:\n reading,gaming,coding", + "expected": null, + "shouldError": true, + "specSection": "6" + }, { "name": "throws on bracket length with leading zeros in strict mode", "input": "items[03]: a,b,c", @@ -325,7 +338,7 @@ }, { "name": "throws on keyed header without a field list in strict mode", - "input": "m[2:]:\n a: 1\n b: 2", + "input": "m[0:]:", "expected": null, "shouldError": true, "options": { @@ -347,7 +360,7 @@ }, { "name": "throws on keyed marker with leading-zero length in strict mode", - "input": "m[03:]{v}:\n a: 1\n b: 2", + "input": "m[02:]{v}:\n a: 1\n b: 2", "expected": null, "shouldError": true, "options": { @@ -389,7 +402,7 @@ }, { "name": "throws on a line without an unquoted colon at entry depth in strict mode", - "input": "m[2:]{v}:\n a: 1\n 5", + "input": "m[1:]{v}:\n a: 1\n 5", "expected": null, "shouldError": true, "options": { @@ -432,6 +445,18 @@ "shouldError": true, "specSection": "6" }, + { + "name": "throws on a keyless list-form header in object field position in non-strict mode", + "input": "a:\n [2]:\n - 1\n - 2", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "6", + "note": "Read as a key-value line, the header's list items become scalar lines (§5.2), so no non-strict leniency yields a value", + "minSpecVersion": "4.2" + }, { "name": "throws on keyless array header after a depth-0 field", "input": "a: 1\n[2]: x,y", @@ -447,6 +472,17 @@ "shouldError": true, "specSection": "6" }, + { + "name": "throws on keyless fields-bearing header as list item in non-strict mode", + "input": "items[1]:\n - [1]{x}:\n 1", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "6", + "minSpecVersion": "4.2" + }, { "name": "throws on inline content after tabular header", "input": "items[2]{a,b}: 1,2", @@ -455,6 +491,18 @@ "specSection": "6", "note": "A fields-bearing header carries no inline content; decoding the values as an inline array would silently drop the field list" }, + { + "name": "throws on a row under a fields-bearing header with inline content in non-strict mode", + "input": "b[1]{c}: x\n y", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "6", + "note": "Read as a key-value line, the row becomes an over-indented scalar line (§5.2), so no non-strict leniency yields a value", + "minSpecVersion": "4.2" + }, { "name": "throws on inline content after root tabular header", "input": "[2]{a,b}: 1,2", @@ -503,6 +551,17 @@ "note": "A scalar line outside root primitive position is an error in strict and non-strict mode alike", "minSpecVersion": "4.1" }, + { + "name": "throws on a bare token line among list-item fields in non-strict mode", + "input": "items[1]:\n - a: 1\n b", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "5.2", + "minSpecVersion": "4.1" + }, { "name": "throws on the removed [#N] length marker in a root header in strict mode", "input": "[#2]: a,b", @@ -664,6 +723,18 @@ "specSection": "6", "minSpecVersion": "4.2" }, + { + "name": "throws on content after a nested field group in non-strict mode", + "input": "a[1]{b{c}d}:\n 1", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "6", + "note": "Not a header (§6 grammar), so the row is a scalar line under a key-value line (§5.2)", + "minSpecVersion": "4.2" + }, { "name": "throws on a \\u escape with a sign before its hex digits", "input": "val: \"\\u+0ab\"", @@ -671,6 +742,13 @@ "shouldError": true, "specSection": "7.1" }, + { + "name": "throws on a \\u escape with non-ASCII digits", + "input": "val: \"\\u\u0660\u0660\u0664\u0661\"", + "expected": null, + "shouldError": true, + "specSection": "7.1" + }, { "name": "throws on an escaped lone low surrogate", "input": "val: \"\\udc00\"", diff --git a/tests/fixtures/encode/delimiters.json b/tests/fixtures/encode/delimiters.json index b8fbc61..24c80d5 100644 --- a/tests/fixtures/encode/delimiters.json +++ b/tests/fixtures/encode/delimiters.json @@ -78,9 +78,10 @@ { "name": "encodes nested arrays with pipe delimiter", "input": { - "pairs": [["a", "b"], ["c", "d"]] + "pairs": [["a", "b"], ["c", "d"]], + "items": [{ "k": [[1, { "x": 1 }]] }] }, - "expected": "pairs[2|]:\n - [2|]: a|b\n - [2|]: c|d", + "expected": "pairs[2|]:\n - [2|]: a|b\n - [2|]: c|d\nitems[1|]:\n - k[1|]:\n - [2|]:\n - 1\n - x: 1", "options": { "delimiter": "|" }, @@ -197,6 +198,17 @@ }, "specSection": "11.1", "minSpecVersion": "4.2" + }, + { + "name": "keeps a root string containing only a non-document delimiter unquoted", + "input": "a,b", + "expected": "a,b", + "options": { + "delimiter": "|" + }, + "specSection": "11.1", + "note": "A root primitive is quoted against the document delimiter (§11.1)", + "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/encode/objects-keyed.json b/tests/fixtures/encode/objects-keyed.json index d8841f5..3b028d5 100644 --- a/tests/fixtures/encode/objects-keyed.json +++ b/tests/fixtures/encode/objects-keyed.json @@ -50,13 +50,14 @@ "name": "uses the active delimiter in keyed headers and entry-row cells", "input": { "servers": { - "alpha": { "host": "a.example.com", "port": 8080 }, - "beta": { "host": "b.example.com", "port": 9090 } + "alpha": { "host": "a.example.com", "port": 8080, "note": "x|y" }, + "beta": { "host": "b.example.com", "port": 9090, "note": "p,q" } } }, "options": { "delimiter": "|" }, - "expected": "servers[2:|]{host|port}:\n alpha: a.example.com|8080\n beta: b.example.com|9090", + "expected": "servers[2:|]{host|port|note}:\n alpha: a.example.com|8080|\"x|y\"\n beta: b.example.com|9090|p,q", "specSection": "9.5", + "minSpecVersion": "4.2", "note": "The keyed marker precedes the delimiter symbol: [2:|]" }, { diff --git a/tests/fixtures/encode/objects.json b/tests/fixtures/encode/objects.json index 7b5b145..ad6271f 100644 --- a/tests/fixtures/encode/objects.json +++ b/tests/fixtures/encode/objects.json @@ -127,11 +127,12 @@ "specSection": "7.3" }, { - "name": "quotes key with leading hyphen", + "name": "quotes keys with a leading or interior hyphen", "input": { - "-lead": 1 + "-lead": 1, + "a-b": 2 }, - "expected": "\"-lead\": 1", + "expected": "\"-lead\": 1\n\"a-b\": 2", "specSection": "7.3" }, { @@ -267,14 +268,17 @@ "specSection": "8" }, { - "name": "keeps strings that only resemble numbers or literals unquoted", + "name": "keeps strings that only resemble numbers or literals, a leading byte-order mark, or DEL unquoted", "input": { "a": ".5", "b": "NaN", - "c": "True" + "c": "True", + "d": "\ufeffx", + "e": "a\u007fb" }, - "expected": "a: .5\nb: NaN\nc: True", + "expected": "a: .5\nb: NaN\nc: True\nd: \ufeffx\ne: a\u007fb", "specSection": "7.2", + "note": "Only a root primitive starting with U+FEFF is quoted, and only U+0000–U+001F count as controls (§7.1, §7.2)", "minSpecVersion": "4.2" } ] diff --git a/tests/fixtures/encode/primitives.json b/tests/fixtures/encode/primitives.json index d53756d..1891165 100644 --- a/tests/fixtures/encode/primitives.json +++ b/tests/fixtures/encode/primitives.json @@ -71,10 +71,11 @@ "specSection": "7.1" }, { - "name": "encodes string with U+0004 control character via \\uXXXX", - "input": { "val": "a\u0004b" }, - "expected": "val: \"a\\u0004b\"", - "specSection": "7.1" + "name": "escapes U+0004 via \\uXXXX and keeps U+2028 literal in quoted strings", + "input": { "val": "a\u0004b", "sep": "a:\u2028b" }, + "expected": "val: \"a\\u0004b\"\nsep: \"a:\u2028b\"", + "specSection": "7.1", + "minSpecVersion": "4.2" }, { "name": "escapes backslash in string", @@ -197,8 +198,8 @@ }, { "name": "encodes floats with their shortest round-trip digits", - "input": [0.3333333333333333, 0.30000000000000004], - "expected": "[2]: 0.3333333333333333,0.30000000000000004", + "input": [0.3333333333333333, 0.30000000000000004, -0.000001], + "expected": "[3]: 0.3333333333333333,0.30000000000000004,-0.000001", "specSection": "2" }, { diff --git a/tests/test_api.py b/tests/test_api.py index 9b914b0..5e1eecb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -27,7 +27,7 @@ def test_public_names() -> None: "loads", ] ) - assert toon_format.__toon_spec__ == "4.2" + assert toon_format.__toon_spec__ == "4.3" assert re.fullmatch(r"\d+\.\d+\.\d+.*", toon_format.__version__) From f2740837ef6781628e08dd46065627fd357f6d26 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 16:28:10 +0200 Subject: [PATCH 02/13] fix(decode): never treat a line without an unquoted colon as a header --- CHANGELOG.md | 2 ++ src/toon_format/_decoder.py | 14 +++++--------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d4daa7..b67325b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ All notable changes to this project are documented here. The format follows colon and that fails the header grammar is an error, also when its only colon sits in the bracket segment or field list (`a[1:`, `a[2:]{x}`). With `strict=False` it is still a key-value line. +- A line without an unquoted colon is never a header, so `a[2]{x,x}` decodes + as a string instead of raising a duplicate field name error in strict mode. ## [1.1.0] - 2026-10-06 diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index c1f8fe2..3a2e35b 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -348,8 +348,8 @@ def classify(self, content: str, line: _Line) -> _Header | _KeyValue | _Scalar: end = _quoted_end(content, 0) if end < 0: raise self.error("Unterminated string", line) - if content.startswith("[", end): - return self.header(content, end, colon, line) + if colon >= 0 and content.startswith("[", end): + return self.header(content, end, line) after = content[end:].lstrip(" ") if not after: return _Scalar(content) @@ -357,17 +357,15 @@ def classify(self, content: str, line: _Line) -> _Header | _KeyValue | _Scalar: raise self.error("Unexpected characters after closing quote", line) return _KeyValue(self.quoted(content[:end], line), after[1:].strip(" ")) bracket = _find_unquoted(content, "[") - if bracket >= 0 and (colon < 0 or bracket < colon): - return self.header(content, bracket, colon, line) + if 0 <= bracket < colon: + return self.header(content, bracket, line) if colon >= 0: return _KeyValue( content[:colon].strip(" "), content[colon + 1 :].strip(" ") ) return _Scalar(content) - def header( - self, content: str, bracket: int, colon: int, line: _Line - ) -> _Header | _KeyValue | _Scalar: + def header(self, content: str, bracket: int, line: _Line) -> _Header | _KeyValue: """Parse a header candidate whose bracket segment starts at ``bracket``.""" match = _BRACKET.match(content, bracket) try: @@ -389,8 +387,6 @@ def header( if fields is not None and rest: raise _Malformed("Unexpected content after a header with a field list") except _Malformed as exc: - if colon < 0: - return _Scalar(content) # without a colon it is a scalar line (§5.2) if self.strict: raise self.error(str(exc), line) from None return self.literal_key_value(content, line) From 8869559209247b6b5d31ddb01806bc552808d58f Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 18:45:38 +0200 Subject: [PATCH 03/13] fix(decode): reject any whitespace before a bracket segment or nested field group --- CHANGELOG.md | 3 +++ src/toon_format/_decoder.py | 4 ++-- tests/test_decode.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b67325b..25eb56c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ All notable changes to this project are documented here. The format follows `strict=False` it is still a key-value line. - A line without an unquoted colon is never a header, so `a[2]{x,x}` decodes as a string instead of raising a duplicate field name error in strict mode. +- Any Unicode whitespace, such as U+00A0, between a key and its bracket + segment or between a field name and its nested field group is a header + error, not only a space or tab. ## [1.1.0] - 2026-10-06 diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 3a2e35b..0669abf 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -371,7 +371,7 @@ def header(self, content: str, bracket: int, line: _Line) -> _Header | _KeyValue try: if match is None: raise _Malformed("Malformed bracket segment") - if content[bracket - 1 : bracket] in (" ", "\t"): + if content[bracket - 1 : bracket].isspace(): raise _Malformed("Whitespace between a key and its bracket segment") delimiter = match.group(3) or "," pos = match.end() @@ -433,7 +433,7 @@ def field_list( pos += 1 nested = None if text.startswith("{", pos): - if text[pos - 1] == " ": + if text[pos - 1].isspace(): raise _Malformed( "Whitespace between a field name and its nested field group" ) diff --git a/tests/test_decode.py b/tests/test_decode.py index 3d675a5..80ab7aa 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -270,6 +270,7 @@ def test_integer_beyond_the_conversion_limit() -> None: ("l[1]:\n - [1]{v}:", {"l": [{"[1]{v}": {}}]}), ("t[1]{a|b}:", {"t[1]{a|b}": {}}), ("t[1]{a{b}c}:", {"t[1]{a{b}c}": {}}), + ("t[1]{a\u00a0{b}}:", {"t[1]{a\u00a0{b}}": {}}), ], ) def test_non_strict_header_fall_through(text: str, expected: Any) -> None: From 59727423148e28b52e87d6d06e1b93f3eea59629 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 19:17:51 +0200 Subject: [PATCH 04/13] fix(decode): treat an invalid escape in a field name as a header syntax error --- src/toon_format/_decoder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 0669abf..87d1e72 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -416,7 +416,11 @@ def field_list( end = _quoted_end(text, pos) if end < 0: raise _Malformed("Unterminated field name") - name = self.unescape(text[pos + 1 : end - 1], line) + try: + name = self.unescape(text[pos + 1 : end - 1], line) + except ToonDecodeError as exc: + # A bad escape fails the header grammar like any other (§6). + raise _Malformed(exc.msg) from None pos = end else: start = pos From bc6b240f746bb32be53a7677649e995a39fa269c Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Wed, 7 Oct 2026 09:46:16 +0200 Subject: [PATCH 05/13] =?UTF-8?q?feat(decode):=20limit=20strict=3DFalse=20?= =?UTF-8?q?to=20the=20five=20recoveries=20of=20=C2=A714.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 16 +++- src/toon_format/_decoder.py | 147 +++++++++++++----------------------- tests/test_decode.py | 23 +----- 3 files changed, 65 insertions(+), 121 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25eb56c..714f53a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,18 @@ All notable changes to this project are documented here. The format follows - Support for [TOON specification 4.3](https://github.com/toon-format/spec/blob/v4.3.0/SPEC.md): all official fixtures of spec `v4.3.0` pass, and `toon_format.__toon_spec__` is `"4.3"`. Encoder output is unchanged. -- In strict mode, a line whose first unquoted `[` precedes its first unquoted - colon and that fails the header grammar is an error, also when its only - colon sits in the bracket segment or field list (`a[1:`, `a[2:]{x}`). With - `strict=False` it is still a key-value line. +- A line whose first unquoted `[` precedes its first unquoted colon and that + fails the header grammar is an error, also when its only colon sits in the + bracket segment or field list (`a[1:`, `a[2:]{x}`). +- `strict=False` keeps five recoveries only: declared lengths are advisory, + duplicate keys and repeated field names resolve last-write-wins, an + indentation tab counts as one level and uneven spaces round down, blank lines + inside arrays and keyed objects are ignored, and a block whose first line + starts too deep takes that depth. Everything else is an error in both modes: + a malformed or misplaced header (`items [2]: a,b`, a keyless `[2]: x,y` + under a field), a row or entry row with the wrong number of cells, an + over-indented line, an entry-depth line without a colon, and content after a + root array or keyed object. - A line without an unquoted colon is never a header, so `a[2]{x,x}` decodes as a string instead of raising a duplicate field name error in strict mode. - Any Unicode whitespace, such as U+00A0, between a key and its bracket diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 87d1e72..7385168 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -29,7 +29,6 @@ _QUOTED = re.compile(r'"(?:[^"\\]|\\.)*"', re.DOTALL) _HEX4 = re.compile(r"[0-9A-Fa-f]{4}") _SIMPLE_ESCAPES = {"\\": "\\", '"': '"', "n": "\n", "r": "\r", "t": "\t"} -_MISSING = object() def decode( @@ -139,10 +138,6 @@ class _Scalar: token: str -class _Malformed(Exception): - """A header that fails the grammar of §6; may fall through in non-strict mode.""" - - def _is_list_item(content: str) -> bool: return content == "-" or content.startswith("- ") @@ -365,42 +360,32 @@ def classify(self, content: str, line: _Line) -> _Header | _KeyValue | _Scalar: ) return _Scalar(content) - def header(self, content: str, bracket: int, line: _Line) -> _Header | _KeyValue: - """Parse a header candidate whose bracket segment starts at ``bracket``.""" + def header(self, content: str, bracket: int, line: _Line) -> _Header: + """Parse a header whose bracket segment starts at ``bracket`` (§6).""" match = _BRACKET.match(content, bracket) - try: - if match is None: - raise _Malformed("Malformed bracket segment") - if content[bracket - 1 : bracket].isspace(): - raise _Malformed("Whitespace between a key and its bracket segment") - delimiter = match.group(3) or "," - pos = match.end() - fields = None - if content.startswith("{", pos): - fields, pos = self.field_list(content, pos, delimiter, line) - if not content.startswith(":", pos): - raise _Malformed("Expected ':' after the array header") - rest = content[pos + 1 :].strip(" ") - keyed = bool(match.group(2)) - if keyed and fields is None: - raise _Malformed("A keyed header requires a field list") - if fields is not None and rest: - raise _Malformed("Unexpected content after a header with a field list") - except _Malformed as exc: - if self.strict: - raise self.error(str(exc), line) from None - return self.literal_key_value(content, line) + if match is None: + raise self.error("Malformed bracket segment", line) + if content[bracket - 1 : bracket].isspace(): + raise self.error("Whitespace between a key and its bracket segment", line) + delimiter = match.group(3) or "," + pos = match.end() + fields = None + if content.startswith("{", pos): + fields, pos = self.field_list(content, pos, delimiter, line) + if not content.startswith(":", pos): + raise self.error("Expected ':' after the array header", line) + rest = content[pos + 1 :].strip(" ") + keyed = bool(match.group(2)) + if keyed and fields is None: + raise self.error("A keyed header requires a field list", line) + if fields is not None and rest: + raise self.error( + "Unexpected content after a header with a field list", line + ) key_text = content[:bracket] key = self.key(key_text, line) if key_text else None return _Header(key, int(match.group(1)), keyed, delimiter, fields, rest) - def literal_key_value(self, content: str, line: _Line) -> _KeyValue: - """Read a rejected header line as a key-value line (§5.2, §6).""" - colon = _find_unquoted(content, ":") - return _KeyValue( - self.key(content[:colon], line), content[colon + 1 :].strip(" ") - ) - def field_list( self, text: str, pos: int, delimiter: str, line: _Line ) -> tuple[Fields, int]: @@ -415,12 +400,8 @@ def field_list( if text.startswith('"', pos): end = _quoted_end(text, pos) if end < 0: - raise _Malformed("Unterminated field name") - try: - name = self.unescape(text[pos + 1 : end - 1], line) - except ToonDecodeError as exc: - # A bad escape fails the header grammar like any other (§6). - raise _Malformed(exc.msg) from None + raise self.error("Unterminated field name", line) + name = self.unescape(text[pos + 1 : end - 1], line) pos = end else: start = pos @@ -432,14 +413,15 @@ def field_list( pos += 1 name = text[start:pos].strip(" ") if not name: - raise _Malformed("Empty field name in field list") + raise self.error("Empty field name in field list", line) while pos < n and text[pos] == " ": pos += 1 nested = None if text.startswith("{", pos): if text[pos - 1].isspace(): - raise _Malformed( - "Whitespace between a field name and its nested field group" + raise self.error( + "Whitespace between a field name and its nested field group", + line, ) nested, pos = self.field_list(text, pos, delimiter, line) while pos < n and text[pos] == " ": @@ -449,25 +431,26 @@ def field_list( seen.add(name) fields.append((name, nested)) if pos >= n: - raise _Malformed("Unmatched '{' in field list") + raise self.error("Unmatched '{' in field list", line) c = text[pos] pos += 1 if c == "}": return tuple(fields), pos if c != delimiter: if c in DELIMITERS: - raise _Malformed( - "Field list delimiter does not match the bracket segment" + raise self.error( + "Field list delimiter does not match the bracket segment", line ) - raise _Malformed("Malformed field list") + raise self.error("Malformed field list", line) # -- document (§5) ------------------------------------------------------ def document(self) -> Any: - while (first := self.peek()) is not None and first.depth > 0: - self.orphan(first) # an indented first line is over-indented (§5, §8) + first = self.peek() if first is None: return self.finish({}) # empty document (§5) + if first.depth > 0: + raise self.over_indented(first) # an indented first line (§5, §8) if first.content == "[]": self.take() self.end_of_root() @@ -487,12 +470,7 @@ def end_of_root(self) -> None: """A root array or keyed object spans the whole document (§5).""" line = self.peek() if line is not None: - if self.strict: - raise self.error("Unexpected content after the root value", line) - for trailing in self.lines[self.pos :]: - if isinstance(self.classify(trailing.content, trailing), _Scalar): - raise self.error("Unexpected bare value", trailing) - self.pos = len(self.lines) + raise self.error("Unexpected content after the root value", line) # -- objects (§8) ------------------------------------------------------- @@ -502,34 +480,24 @@ def object_body( """Parse the fields at ``depth`` of an object opened at ``parent`` (§8).""" while (line := self.peek()) is not None and line.depth > parent: if line.depth != depth: - self.orphan(line) - continue + raise self.over_indented(line) self.take() self.field(line, obj, depth) return obj - def orphan(self, line: _Line) -> None: - """Handle a line deeper than its scope that no line opened (§8, §14.2).""" - if self.strict: - raise self.error("Unexpected indentation", line) - if isinstance(self.classify(line.content, line), _Scalar): - raise self.error("Unexpected bare value", line) - self.take() + def over_indented(self, line: _Line) -> ToonDecodeError: + """A line deeper than its scope that no line opened (§8, §14.2).""" + return self.error("Unexpected indentation", line) def field(self, line: _Line, obj: dict[str, Any], depth: int) -> None: kind = self.classify(line.content, line) if isinstance(kind, _Header): if kind.key is None: - if self.strict: - raise self.error( - "A header without a key is only valid at the root", line - ) - kind = self.literal_key_value(line.content, line) - else: - self.assign( - obj, kind.key, self.header_value(kind, line, depth + 1), line + raise self.error( + "A header without a key is only valid at the root", line ) - return + self.assign(obj, kind.key, self.header_value(kind, line, depth + 1), line) + return if isinstance(kind, _Scalar): raise self.error("Expected 'key: value'", line) self.assign(obj, kind.key, self.field_value(kind.rest, line, depth), line) @@ -565,7 +533,7 @@ def check_count(self, count: int, header: _Header, line: _Line) -> None: def check_width(self, cells: list[Any], fields: Fields, line: _Line) -> None: width = _leaf_count(fields) - if self.strict and len(cells) != width: + if len(cells) != width: raise self.error(f"Expected {width} values but found {len(cells)}", line) def items(self, header: _Header, line: _Line, depth: int, parent: int) -> list[Any]: @@ -575,8 +543,7 @@ def items(self, header: _Header, line: _Line, depth: int, parent: int) -> list[A try: while (item := self.peek()) and item.depth > parent: if item.depth != depth: - self.orphan(item) - continue + raise self.over_indented(item) if not _is_list_item(item.content): break self.take() @@ -600,13 +567,11 @@ def item(self, line: _Line, depth: int) -> Any: if isinstance(kind, _Scalar): return self.primitive(rest, line) if isinstance(kind, _Header) and kind.key is None: - if kind.fields is None: - return self.header_value(kind, line, depth + 1) - if self.strict: + if kind.fields is not None: raise self.error( "A header with a field list cannot be a list item", line ) - kind = self.literal_key_value(rest, line) + return self.header_value(kind, line, depth + 1) # An object whose first field sits on the hyphen line at depth + 1. obj: dict[str, Any] = {} if isinstance(kind, _Header): @@ -625,8 +590,7 @@ def rows(self, header: _Header, line: _Line, depth: int, parent: int) -> list[An try: while (row := self.peek()) and row.depth > parent: if row.depth != depth: - self.orphan(row) - continue + raise self.over_indented(row) colon = _find_unquoted(row.content, ":") if colon >= 0: split = _find_unquoted(row.content, delimiter) @@ -652,17 +616,14 @@ def entries(self, header: _Header, line: _Line, depth: int, parent: int) -> Any: try: while (entry := self.peek()) and entry.depth > parent: if entry.depth != depth: - self.orphan(entry) - continue + raise self.over_indented(entry) self.take() if not count: self.spans += 1 count += 1 colon = _find_unquoted(entry.content, ":") if colon < 0: - if self.strict: - raise self.error("Expected 'key: values' entry row", entry) - continue + raise self.error("Expected 'key: values' entry row", entry) key = self.key(entry.content[:colon], entry) cells = self.values( entry.content[colon + 1 :].strip(" "), header.delimiter, entry @@ -680,11 +641,7 @@ def materialize(self, fields: Fields, cells: Iterator[Any]) -> Any: """Build one object from row cells, walking the field list depth-first.""" obj: dict[str, Any] = {} for name, nested in fields: - value = ( - next(cells, _MISSING) - if nested is None - else self.materialize(nested, cells) + obj[name] = ( + next(cells) if nested is None else self.materialize(nested, cells) ) - if value is not _MISSING: - obj[name] = value return self.finish(obj) diff --git a/tests/test_decode.py b/tests/test_decode.py index 80ab7aa..008b40b 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -181,10 +181,7 @@ def test_deep_nesting_raises_decode_error() -> None: ("a:\n\tb: 1", {"a": {"b": 1}}), # a tab is one level ("a:\n b: 1\n c: 2", {"a": {"b": 1, "c": 2}}), # depth jump ("items[2]:\n - x\n - y", {"items": ["x", "y"]}), - ("a: 1\n b: 2\nc: 3", {"a": 1, "c": 3}), # over-indented line is skipped ("a: 1\nb: 2\na: 3", {"a": 3, "b": 2}), # last write wins, first position kept - ("t[2]{x,y}:\n 1\n 2,3,4", {"t": [{"x": 1}, {"x": 2, "y": 3}]}), - ("[2]: a\njunk: 1", ["a"]), # trailing content after a root array is ignored ], ) def test_non_strict(text: str, expected: Any) -> None: @@ -240,10 +237,8 @@ def test_row_or_key_value_at_row_depth() -> None: # §9.3: a delimiter before the first colon keeps the line a row... assert toon_format.loads("t[1]{x,y}:\n a,b:c") == {"t": [{"x": "a", "y": "b:c"}]} # ...and a colon first ends the rows, leaving the line in no scope. - text = "t[1]{x}:\n 1\n n: 2" with pytest.raises(toon_format.ToonDecodeError, match="Unexpected indentation"): - toon_format.loads(text) - assert toon_format.loads(text, strict=False) == {"t": [{"x": 1}]} + toon_format.loads("t[1]{x}:\n 1\n n: 2") def test_hyphen_outside_a_list_is_part_of_the_key() -> None: @@ -263,21 +258,5 @@ def test_integer_beyond_the_conversion_limit() -> None: toon_format.loads("n: " + "9" * 5000) -@pytest.mark.parametrize( - ("text", "expected"), - [ - ("a:\n [2]: x,y", {"a": {"[2]": "x,y"}}), - ("l[1]:\n - [1]{v}:", {"l": [{"[1]{v}": {}}]}), - ("t[1]{a|b}:", {"t[1]{a|b}": {}}), - ("t[1]{a{b}c}:", {"t[1]{a{b}c}": {}}), - ("t[1]{a\u00a0{b}}:", {"t[1]{a\u00a0{b}}": {}}), - ], -) -def test_non_strict_header_fall_through(text: str, expected: Any) -> None: - with pytest.raises(toon_format.ToonDecodeError): - toon_format.loads(text) - assert toon_format.loads(text, strict=False) == expected - - def test_unterminated_quote_in_a_field_list_hides_the_colon() -> None: assert toon_format.loads('t[1]{"x}:') == 't[1]{"x}:' From b6bcfe3d9a00ac80df50260af54b24dd3a7c9cdc Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Wed, 7 Oct 2026 09:46:46 +0200 Subject: [PATCH 06/13] fix(decode): treat only space and tab as whitespace before a bracket segment or nested field group --- CHANGELOG.md | 3 --- src/toon_format/_decoder.py | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 714f53a..17086a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,6 @@ All notable changes to this project are documented here. The format follows root array or keyed object. - A line without an unquoted colon is never a header, so `a[2]{x,x}` decodes as a string instead of raising a duplicate field name error in strict mode. -- Any Unicode whitespace, such as U+00A0, between a key and its bracket - segment or between a field name and its nested field group is a header - error, not only a space or tab. ## [1.1.0] - 2026-10-06 diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 7385168..5528c88 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -365,7 +365,7 @@ def header(self, content: str, bracket: int, line: _Line) -> _Header: match = _BRACKET.match(content, bracket) if match is None: raise self.error("Malformed bracket segment", line) - if content[bracket - 1 : bracket].isspace(): + if content[bracket - 1 : bracket] in (" ", "\t"): # SP and HTAB only (§1.2) raise self.error("Whitespace between a key and its bracket segment", line) delimiter = match.group(3) or "," pos = match.end() @@ -418,7 +418,7 @@ def field_list( pos += 1 nested = None if text.startswith("{", pos): - if text[pos - 1].isspace(): + if text[pos - 1] == " ": raise self.error( "Whitespace between a field name and its nested field group", line, From eb08743f6bab440dd2b448979b115b68338e94af Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Wed, 7 Oct 2026 09:47:30 +0200 Subject: [PATCH 07/13] docs: describe the five recoveries strict=False keeps --- AGENTS.md | 17 +++++++++-------- docs/usage.md | 9 ++++++--- src/toon_format/_api.py | 8 +++++--- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5fdc0ee..d54ba11 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -101,14 +101,15 @@ Set `HYPOTHESIS_PROFILE=ci` to run the property tests with more examples, as CI - Comment lines and blank lines are handled while splitting lines, before any structural decision (§5.1, §12). Line numbers in errors always refer to the original document. -- Line classification follows §5.2. In strict mode a malformed header is an - error; in non-strict mode it falls back to a key-value line with a literal - key (§6). -- `strict=False` relaxes only what §14 lists as strict-only: counts and - widths, blank lines in header spans, indentation, over-indented lines, - duplicate keys (last write wins, first position kept), malformed headers, - and trailing content after a root array. Errors marked "any mode" in §14 stay - errors. +- Line classification follows §5.2. A malformed header is an error in both + modes (§6). +- Whitespace is space and tab only (§1.2): never decide syntax with + `str.isspace()`, an argument-less `strip()`, or `\s`. +- `strict=False` applies exactly the five recoveries of §14.4: counts are + advisory, duplicate keys and repeated field names resolve last-write-wins + (first position kept), indentation depth is tabs plus floored spaces, blank + lines in header spans are ignored, and a jumped first line sets its scope's + depth. Every other §14 condition is an error in both modes. - A declared `[N]` never ends or truncates a scope; it is only compared with what the scope contains. - Numbers follow the §4 grammar, never Python's `int()`/`float()` grammar, diff --git a/docs/usage.md b/docs/usage.md index 0cc4790..f5c13f6 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -90,9 +90,12 @@ toon_format.load(fp, **options) -> Any ``` `s` may also be `bytes`, and `fp` a binary file. By default the decoder -enforces every check of the specification: declared lengths, row widths, -indentation, blank lines inside arrays, and duplicate keys. `strict=False` -tolerates them, which helps with hand-written or LLM-generated documents. +enforces every check of the specification. `strict=False` tolerates five of +them, which helps with hand-written or LLM-generated documents: a declared +length that differs from the actual count, duplicate keys (the last one wins), +tab or uneven indentation, blank lines inside arrays and keyed objects, and a +block that starts too deep. Every other check, such as a row's number of cells +or a header's syntax, applies in both modes. The hooks work as in `json.loads`: ```python diff --git a/src/toon_format/_api.py b/src/toon_format/_api.py index 298ba4f..f94aa68 100644 --- a/src/toon_format/_api.py +++ b/src/toon_format/_api.py @@ -114,9 +114,11 @@ def loads( Args: s: The document. Bytes are decoded as UTF-8; ill-formed UTF-8 is an error. - strict: Enforce every check of §14. With ``strict=False``, length and - width mismatches, blank lines inside arrays, irregular - indentation, and duplicate keys (last one wins) are tolerated. + strict: Enforce every check of §14. With ``strict=False``, declared + lengths are advisory, and duplicate keys (last one wins), tab or + uneven indentation, blank lines inside arrays and keyed objects, + and a block that starts too deep are tolerated (§14.4); every + other check applies. indent_size: Spaces per indentation level. parse_float: Called with the text of every number that has a fraction or an exponent, as in :func:`json.loads`. Use From b36230a4c92188da518104b24fbf0698b342eb36 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Wed, 7 Oct 2026 10:49:36 +0200 Subject: [PATCH 08/13] feat(decode): reject an out-of-range number with strict=False too --- CHANGELOG.md | 2 ++ docs/data-types.md | 10 +++++----- src/toon_format/_decoder.py | 2 +- tests/test_decode.py | 1 - 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17086a3..c25886d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ All notable changes to this project are documented here. The format follows under a field), a row or entry row with the wrong number of cells, an over-indented line, an entry-depth line without a colon, and content after a root array or keyed object. +- A number beyond the double range is an error with `strict=False` too, instead + of decoding to `±inf`. - A line without an unquoted colon is never a header, so `a[2]{x,x}` decodes as a string instead of raising a duplicate field name error in strict mode. diff --git a/docs/data-types.md b/docs/data-types.md index 5f143c0..bf08c87 100644 --- a/docs/data-types.md +++ b/docs/data-types.md @@ -37,11 +37,11 @@ asks implementations to document the following choices. shortest representation that round-trips, in plain decimal between 1e-6 and 1e21 and in exponent form (`1e+21`, `1e-7`) outside it. Integral floats from 2⁵³ up are written with their exact digits, so that they decode to an equal -`int`. A decoded number beyond the double range is an error in strict mode and -`±inf` otherwise. `Decimal` input keeps every digit, and `parse_float=Decimal` -keeps them when decoding. A `Decimal` beyond the double range, such as -`Decimal("1e400")`, therefore encodes but does not decode back with the -default strict decoder: pass `parse_float=Decimal` to read it. +`int`. A decoded number beyond the double range is an error in both modes. +`Decimal` input keeps every digit, and `parse_float=Decimal` keeps them when +decoding. A `Decimal` beyond the double range, such as `Decimal("1e400")`, +therefore encodes but does not decode back by default: pass +`parse_float=Decimal` to read it. **Strings.** A string holding an unpaired surrogate cannot be encoded and raises `ValueError`. A root string starting with U+FEFF is quoted, because a diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 5528c88..4efea4c 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -324,7 +324,7 @@ def number(self, token: str, line: _Line) -> Any: if self.parse_float is not None: return self.parse_float(token) value = float(token) - if math.isinf(value) and self.strict: + if math.isinf(value): raise self.error(f"Number {token} is out of range", line) return value or 0.0 # -0 decodes to 0 (§4) diff --git a/tests/test_decode.py b/tests/test_decode.py index 008b40b..3ff8483 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -40,7 +40,6 @@ def test_negative_zero_decodes_to_positive_zero() -> None: def test_out_of_range_floats() -> None: with pytest.raises(toon_format.ToonDecodeError, match="out of range"): toon_format.loads("x: 1e400") - assert toon_format.loads("x: -1e400", strict=False) == {"x": float("-inf")} def test_parse_float_and_parse_int() -> None: From 4627265e296aa3a58b743a7777a2d4369c326561 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Wed, 7 Oct 2026 10:49:41 +0200 Subject: [PATCH 09/13] docs: drop the tab-indentation note the spec now defines --- docs/data-types.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/data-types.md b/docs/data-types.md index bf08c87..0f5c984 100644 --- a/docs/data-types.md +++ b/docs/data-types.md @@ -53,8 +53,5 @@ last value, as in `json.loads`. When encoding, keys that become equal once converted to strings (`1` and `"1"`, `True` and `"true"`) raise `ValueError` instead of dropping a value. No key is special: `__proto__` is an ordinary key. -**Tabs in indentation** are an error in strict mode; otherwise each leading tab -counts as one level. - **Nesting** deeper than the interpreter's recursion limit raises `ValueError` when encoding and `ToonDecodeError` when decoding. From c06a4bb1753ccee87eeff72799f8f1e1ff8635ab Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Wed, 7 Oct 2026 10:49:48 +0200 Subject: [PATCH 10/13] =?UTF-8?q?docs:=20point=20the=20non-strict=20recove?= =?UTF-8?q?ry=20comments=20at=20=C2=A714.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/toon_format/_decoder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/toon_format/_decoder.py b/src/toon_format/_decoder.py index 4efea4c..dfe1b5b 100644 --- a/src/toon_format/_decoder.py +++ b/src/toon_format/_decoder.py @@ -82,7 +82,7 @@ def _split_lines(text: str, indent_size: int, strict: bool) -> list[_Line]: if stripped.startswith("#"): continue # comment line (§5.1); lines are adjacent across it content = stripped.rstrip(" ") - # Tabs make a line blank only where they may indent it (§12). + # Tabs make a line blank only where they may indent it (§14.4). if not content or (not strict and not content.strip(" \t")): blank = blank or number continue @@ -92,7 +92,7 @@ def _split_lines(text: str, indent_size: int, strict: bool) -> list[_Line]: raise ToonDecodeError( "Tabs are not allowed in indentation", number, source ) - # Non-strict: a tab counts as one level, spaces as usual (§12). + # Non-strict: a tab counts as one level, spaces as usual (§14.4). run = len(content) - len(content.lstrip(" \t")) spaces += content.count(" ", 0, run) depth = content.count("\t", 0, run) + spaces // indent_size @@ -252,7 +252,7 @@ def finish(self, obj: dict[str, Any]) -> Any: def assign(self, obj: dict[str, Any], key: str, value: Any, line: _Line) -> None: if self.strict and key in obj: raise self.error(f"Duplicate key {key!r}", line) - # Non-strict: last write wins (§14.3); like json.loads, the key keeps + # Non-strict: last write wins (§14.4); like json.loads, the key keeps # the position of its first occurrence. obj[key] = value From da5b8e08b2b00616dcb0aff65e2882b96d134931 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Thu, 8 Oct 2026 17:50:04 +0200 Subject: [PATCH 11/13] test(fixtures): update fixtures to version 4.4.0 --- tests/fixtures/README.md | 2 +- tests/fixtures/decode/arrays-nested.json | 14 +- tests/fixtures/decode/arrays-primitive.json | 6 +- tests/fixtures/decode/arrays-tabular.json | 53 ++--- tests/fixtures/decode/blank-lines.json | 49 ++-- tests/fixtures/decode/comments.json | 5 +- tests/fixtures/decode/delimiters.json | 40 ++-- tests/fixtures/decode/indentation-errors.json | 215 +++++++++++------- tests/fixtures/decode/numbers.json | 14 +- tests/fixtures/decode/objects-keyed.json | 37 +-- tests/fixtures/decode/objects.json | 157 +++++-------- tests/fixtures/decode/primitives.json | 8 +- tests/fixtures/decode/root-form.json | 22 +- tests/fixtures/decode/validation-errors.json | 205 +++++++++++++---- tests/fixtures/decode/whitespace.json | 60 +++-- tests/fixtures/encode/arrays-nested.json | 3 +- tests/fixtures/encode/arrays-primitive.json | 4 +- tests/fixtures/encode/objects.json | 23 +- tests/fixtures/encode/primitives.json | 40 ++-- tests/fixtures/encode/whitespace.json | 2 +- 20 files changed, 575 insertions(+), 384 deletions(-) diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 510ee2e..e39d6a0 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -2,5 +2,5 @@ The `encode/` and `decode/` directories are copied verbatim from the [TOON specification repository](https://github.com/toon-format/spec/tree/main/tests/fixtures) -at tag `v4.3.0`. Do not edit them by hand: run `uv run scripts/update_fixtures.py ` +at tag `v4.4.0`. Do not edit them by hand: run `uv run scripts/update_fixtures.py ` to replace them with another release. diff --git a/tests/fixtures/decode/arrays-nested.json b/tests/fixtures/decode/arrays-nested.json index 34d17d6..e61312b 100644 --- a/tests/fixtures/decode/arrays-nested.json +++ b/tests/fixtures/decode/arrays-nested.json @@ -175,17 +175,20 @@ }, { "name": "keeps every list item when the count mismatches in non-strict mode", - "input": "a[1]:\n - 1\n - 2", + "input": "a[1]:\n - 1\n - 2\nb[3]:\n - x", "expected": { "a": [ 1, 2 + ], + "b": [ + "x" ] }, "options": { "strict": false }, - "specSection": "14.1", + "specSection": "14.4", "note": "A declared [N] never truncates a scope", "minSpecVersion": "4.1" }, @@ -265,16 +268,17 @@ "options": { "strict": false }, - "specSection": "14.1", - "note": "A legacy [0] is a declared length, so its content is decoded (§14.1)", + "specSection": "14.4", + "note": "A legacy [0] is a declared length, so its content is decoded (§14.4)", "minSpecVersion": "4.1" }, { "name": "parses list items shaped like headers without a colon as strings", - "input": "a[2]:\n - [2]\n - [1,2,3]", + "input": "a[3]:\n - [2]\n - a[2]{x,x}\n - [1,2,3]", "expected": { "a": [ "[2]", + "a[2]{x,x}", "[1,2,3]" ] }, diff --git a/tests/fixtures/decode/arrays-primitive.json b/tests/fixtures/decode/arrays-primitive.json index 96978fe..a92dde3 100644 --- a/tests/fixtures/decode/arrays-primitive.json +++ b/tests/fixtures/decode/arrays-primitive.json @@ -124,8 +124,8 @@ "options": { "strict": false }, - "specSection": "14.1", - "note": "A legacy [0] is a declared length, so its content is decoded (§14.1)", + "specSection": "14.4", + "note": "A legacy [0] is a declared length, so its content is decoded (§14.4)", "minSpecVersion": "4.1" }, { @@ -140,7 +140,7 @@ "options": { "strict": false }, - "specSection": "14.1", + "specSection": "14.4", "minSpecVersion": "4.1" }, { diff --git a/tests/fixtures/decode/arrays-tabular.json b/tests/fixtures/decode/arrays-tabular.json index 1292e1e..ece395e 100644 --- a/tests/fixtures/decode/arrays-tabular.json +++ b/tests/fixtures/decode/arrays-tabular.json @@ -95,11 +95,11 @@ "specSection": "9.3" }, { - "name": "parses sibling nested field groups by depth-first cell assignment", - "input": "shipments[1]{id,sender{name,city},receiver{name,city}}:\n s1,ACME,Berlin,Globex,Oslo", + "name": "parses sibling nested field groups by depth-first cell assignment, with a name repeated across levels", + "input": "shipments[1]{id,sender{id,city},receiver{id,city}}:\n s1,ACME,Berlin,Globex,Oslo", "expected": { "shipments": [ - { "id": "s1", "sender": { "name": "ACME", "city": "Berlin" }, "receiver": { "name": "Globex", "city": "Oslo" } } + { "id": "s1", "sender": { "id": "ACME", "city": "Berlin" }, "receiver": { "id": "Globex", "city": "Oslo" } } ] }, "specSection": "9.3" @@ -148,7 +148,7 @@ "specSection": "9.3" }, { - "name": "applies LWW for duplicate field names, bare or grouped, in non-strict mode", + "name": "applies last-write-wins for duplicate field names, bare or grouped, in non-strict mode", "input": "items[1]{a,a,b,b{x}}:\n 1,2,3,4", "expected": { "items": [ @@ -158,16 +158,17 @@ "options": { "strict": false }, - "specSection": "14.3", + "specSection": "14.4", "note": "Duplicate field names yield duplicate sibling keys per row; non-strict resolves last-write-wins" }, { - "name": "accepts a field name outside the encoder unquoted-key pattern", - "input": "items[1]{2key}:\n 1", + "name": "accepts field names outside the encoder unquoted-key pattern", + "input": "items[1]{2key,x:y}:\n 1,2", "expected": { "items": [ { - "2key": 1 + "2key": 1, + "x:y": 2 } ] }, @@ -190,7 +191,7 @@ "options": { "strict": false }, - "specSection": "14.1", + "specSection": "14.4", "note": "A declared [N] never truncates a scope", "minSpecVersion": "4.1" }, @@ -225,41 +226,30 @@ "minSpecVersion": "4.1" }, { - "name": "materializes a nested field group without cells as an empty object in non-strict mode", + "name": "throws on a row missing a nested field group's cells in non-strict mode", "input": "items[1]{a,n{x}}:\n 1", - "expected": { - "items": [ - { - "a": 1, - "n": {} - } - ] - }, + "expected": null, + "shouldError": true, "options": { "strict": false }, "specSection": "14.1", - "minSpecVersion": "4.1" + "minSpecVersion": "4.4" }, { - "name": "drops surplus cells when the width mismatches in non-strict mode", + "name": "throws on a row with more cells than fields in non-strict mode", "input": "items[1]{a}:\n 1,2", - "expected": { - "items": [ - { - "a": 1 - } - ] - }, + "expected": null, + "shouldError": true, "options": { "strict": false }, "specSection": "14.1", - "minSpecVersion": "4.1" + "minSpecVersion": "4.4" }, { - "name": "parses unquoted colon after the delimiter in tabular row as data", - "input": "items[1]{id,note}:\n 1,a:b", + "name": "parses an unquoted colon after the active delimiter in a tabular row as data", + "input": "items[1|]{id|note}:\n 1|a:b", "expected": { "items": [ { @@ -294,8 +284,7 @@ "": 2 } ], - "specSection": "6", - "minSpecVersion": "4.2" + "specSection": "6" }, { "name": "unescapes a quoted field name", diff --git a/tests/fixtures/decode/blank-lines.json b/tests/fixtures/decode/blank-lines.json index cbeaefe..1edd7f2 100644 --- a/tests/fixtures/decode/blank-lines.json +++ b/tests/fixtures/decode/blank-lines.json @@ -1,7 +1,7 @@ { "version": "4.0", "category": "decode", - "description": "Blank line handling – strict mode errors on blank lines inside arrays, accepts blank lines outside arrays", + "description": "Blank line handling – a blank line inside a header span errors in strict mode and is ignored in non-strict mode; blank lines elsewhere are ignored", "tests": [ { "name": "throws on blank line inside list array", @@ -101,15 +101,18 @@ "specSection": "12" }, { - "name": "accepts blank line after array ends, at root and in a nested object", - "input": "items[1]:\n - a\n\nb: 2\nc:\n d[1]:\n - x\n\n e: 3", + "name": "accepts blank lines after list, tabular, and keyed scopes end, at root and in a nested object", + "input": "items[1]:\n - a\n\nb: 2\nc:\n d[1]:\n - x\n\n e: 3\nt[1]{a}:\n 1\n\nm[1:]{v}:\n k: 1\n\nz: 0", "expected": { "items": ["a"], "b": 2, "c": { "d": ["x"], "e": 3 - } + }, + "t": [{ "a": 1 }], + "m": { "k": { "v": 1 } }, + "z": 0 }, "options": { "strict": true @@ -117,8 +120,8 @@ "specSection": "12" }, { - "name": "accepts blank line between nested object fields", - "input": "a:\n b: 1\n\n c: 2", + "name": "accepts blank lines before and between nested object fields", + "input": "a:\n\n b: 1\n\n c: 2", "expected": { "a": { "b": 1, @@ -131,7 +134,22 @@ "specSection": "12" }, { - "name": "ignores blank lines inside list array when strict=false", + "name": "accepts blank lines before and between nested object fields in non-strict mode", + "input": "a:\n\n b: 1\n\n c: 2", + "expected": { + "a": { + "b": 1, + "c": 2 + } + }, + "options": { + "strict": false + }, + "specSection": "12", + "minSpecVersion": "4.4" + }, + { + "name": "ignores blank lines inside list array in non-strict mode", "input": "items[4]:\n - a\n\n - b\n\n\n - c\n - x: 1\n\n y: 2", "expected": { "items": ["a", "b", "c", { "x": 1, "y": 2 }] @@ -139,21 +157,24 @@ "options": { "strict": false }, - "specSection": "12" + "specSection": "14.4", + "minSpecVersion": "4.4" }, { - "name": "ignores blank lines inside tabular array when strict=false", - "input": "items[2]{id,name}:\n 1,Ada\n\n 2,Bob", + "name": "ignores blank lines between tabular rows and entry rows in non-strict mode", + "input": "items[2]{id,name}:\n 1,Ada\n\n 2,Bob\nm[2:]{v}:\n a: 1\n\n b: 2", "expected": { "items": [ { "id": 1, "name": "Ada" }, { "id": 2, "name": "Bob" } - ] + ], + "m": { "a": { "v": 1 }, "b": { "v": 2 } } }, "options": { "strict": false }, - "specSection": "12" + "specSection": "14.4", + "minSpecVersion": "4.4" }, { "name": "throws on blank line between list items after nested tabular rows", @@ -231,8 +252,8 @@ "options": { "strict": false }, - "specSection": "12", - "minSpecVersion": "4.2" + "specSection": "14.4", + "minSpecVersion": "4.4" } ] } diff --git a/tests/fixtures/decode/comments.json b/tests/fixtures/decode/comments.json index 5db101b..a77494d 100644 --- a/tests/fixtures/decode/comments.json +++ b/tests/fixtures/decode/comments.json @@ -104,7 +104,7 @@ "strict": false }, "specSection": "5.1", - "note": "Comment removal applies in both modes; the resulting count mismatch is not an error when strict=false (§14.1)" + "note": "The resulting count mismatch is not an error when strict=false (§14.4)" }, { "name": "throws on tab-indented hash line, which is not a comment", @@ -134,7 +134,8 @@ "strict": false }, "specSection": "5.1", - "note": "Comment removal precedes the §12 tab leniency, so the tab keeps the line out of the pre-pass" + "note": "Comment removal precedes the §14.4 indentation recovery, so the tab keeps the line out of the pre-pass", + "minSpecVersion": "4.4" }, { "name": "parses quoted hash-leading first cell as data, not comment", diff --git a/tests/fixtures/decode/delimiters.json b/tests/fixtures/decode/delimiters.json index 26a7272..494ff15 100644 --- a/tests/fixtures/decode/delimiters.json +++ b/tests/fixtures/decode/delimiters.json @@ -178,28 +178,16 @@ "specSection": "11" }, { - "name": "parses tabular headers with keys containing the active delimiter", - "input": "items[2|]{\"a|b\"}:\n 1\n 2", + "name": "parses tabular headers with keys containing a delimiter", + "input": "items[2|]{\"a|b\"|\"c,d\"}:\n 1|x\n 2|y", "expected": { - "items": [{ "a|b": 1 }, { "a|b": 2 }] + "items": [{ "a|b": 1, "c,d": "x" }, { "a|b": 2, "c,d": "y" }] }, "specSection": "11" }, - { - "name": "falls through to a key-value line on a header delimiter mismatch", - "input": "a[0|]{x,y}:", - "expected": { - "a[0|]{x,y}": {} - }, - "options": { - "strict": false - }, - "specSection": "6", - "minSpecVersion": "4.1" - }, { "name": "treats a delimiter inside a closed or unclosed mid-token quote span as quoted", - "input": "k[2]: a\"b,c\",d\nu[1]: x\"y,z", + "input": "k[2]: a\"b,c\",d\nu[1]: x\"y,z\nv[1]: x\\\"y,z", "expected": { "k": [ "a\"b,c\"", @@ -207,6 +195,9 @@ ], "u": [ "x\"y,z" + ], + "v": [ + "x\\\"y,z" ] }, "specSection": "5.2", @@ -219,6 +210,23 @@ "items": [{ "t": [{ "a": 1, "b": 2 }] }] }, "specSection": "11.2" + }, + { + "name": "matches a delimiter, colon, space, or comment marker followed by a combining mark as syntax", + "input": "a[2]: x,\u0301y\nb:\u0301z\nk: \u0301z\nl[1]:\n - \u0301w\n#\u0301 c", + "expected": { + "a": [ + "x", + "\u0301y" + ], + "b": "\u0301z", + "k": "\u0301z", + "l": [ + "\u0301w" + ] + }, + "specSection": "1.2", + "minSpecVersion": "4.4" } ] } diff --git a/tests/fixtures/decode/indentation-errors.json b/tests/fixtures/decode/indentation-errors.json index 16e4b9e..791636a 100644 --- a/tests/fixtures/decode/indentation-errors.json +++ b/tests/fixtures/decode/indentation-errors.json @@ -1,7 +1,7 @@ { "version": "4.0", "category": "decode", - "description": "Strict mode indentation validation – non-multiple indentation, tab characters, custom indentSizes", + "description": "Indentation validation and its non-strict recoveries – non-multiple indentation, tab characters, custom indentSizes", "tests": [ { "name": "throws on object field with non-multiple indentation (3 spaces with indentSize 2)", @@ -38,11 +38,18 @@ }, { "name": "accepts correct indentation with custom indentSize (4 spaces with indentSize 4)", - "input": "a:\n b: 1", + "input": "a:\n b: 1\nl[2]:\n - b: 1\n c: 2\n - 1", "expected": { "a": { "b": 1 - } + }, + "l": [ + { + "b": 1, + "c": 2 + }, + 1 + ] }, "options": { "indentSize": 4, @@ -62,7 +69,7 @@ }, { "name": "throws on mixed tabs and spaces in indentation", - "input": "a:\n \tb: 1", + "input": "a:\n b:\n \t\tc: 1", "expected": null, "shouldError": true, "options": { @@ -114,19 +121,38 @@ "specSection": "12" }, { - "name": "accepts non-multiple indentation at nested depths when strict=false", - "input": "a:\n b:\n c: 1\n d: 2", + "name": "accepts non-multiple indentation at nested depths in non-strict mode", + "input": "a:\n b:\n c: 1\n d: 2\n e[2]: 1,2", "expected": { "a": { "b": { "c": 1 }, - "d": 2 + "d": 2, + "e": [1, 2] } }, "options": { "indentSize": 2, "strict": false }, - "specSection": "12" + "specSection": "14.4", + "minSpecVersion": "4.4" + }, + { + "name": "counts every tab and space of mixed indentation in non-strict mode", + "input": "a:\n b:\n \t c: 1", + "expected": { + "a": { + "b": { + "c": 1 + } + } + }, + "options": { + "strict": false + }, + "specSection": "14.4", + "note": "One tab plus two spaces is depth 2 at indentSize 2, wherever the tab sits", + "minSpecVersion": "4.4" }, { "name": "throws on depth jump of more than one level", @@ -150,6 +176,13 @@ "shouldError": true, "specSection": "8" }, + { + "name": "throws on a field line indented under a bare hyphen", + "input": "items[1]:\n -\n b: 1", + "expected": null, + "shouldError": true, + "specSection": "8" + }, { "name": "throws on a line indented one level under a primitive field", "input": "a: 1\n b: 2", @@ -174,134 +207,146 @@ "specSection": "8" }, { - "name": "throws on orphan scalar line under a primitive field in non-strict mode", - "input": "a: 1\n hello", + "name": "throws on an over-indented line after keyed entry rows", + "input": "m[1:]{v}:\n a: 1\n b: 2", + "expected": null, + "shouldError": true, + "specSection": "8" + }, + { + "name": "throws on an over-indented line under a primitive field in non-strict mode", + "input": "a: 1\n b: 2", "expected": null, "shouldError": true, "options": { "strict": false }, "specSection": "8", - "note": "The non-strict skip for over-indented lines excludes scalar lines (§5.2)", - "minSpecVersion": "4.1" + "minSpecVersion": "4.4" }, { - "name": "throws on an indented first line", - "input": " a: 1", + "name": "throws on an over-indented line after tabular rows in non-strict mode", + "input": "u[1]{a}:\n 1\n junk: 9", "expected": null, "shouldError": true, + "options": { + "strict": false + }, "specSection": "8", - "minSpecVersion": "4.2" + "minSpecVersion": "4.4" }, { - "name": "throws on a hyphen-leading bare token under a primitive field in non-strict mode", - "input": "a: 1\n - x", + "name": "throws on an over-indented line after keyed entry rows in non-strict mode", + "input": "m[1:]{v}:\n a: 1\n b: 2", "expected": null, "shouldError": true, "options": { "strict": false }, - "specSection": "5.2", + "specSection": "8", + "minSpecVersion": "4.4" + }, + { + "name": "throws on an indented first line", + "input": " a: 1", + "expected": null, + "shouldError": true, + "specSection": "8", "minSpecVersion": "4.2" }, { "name": "adopts the depth of a jumped first line in non-strict mode", - "input": "a:\n b: 1\n c: 2\n d: 3\nitems[2]:\n - 1\n - 2", + "input": "a:\n b:\n x: 1\n c: 2\nitems[2]:\n - 1\n - 2\nt[1]{a}:\n 1\nm[1:]{v}:\n k: 2\nl[1]:\n - k:\n x: 1", "expected": { - "a": { - "b": 1, - "d": 3 - }, - "items": [ - 1, - 2 - ] + "a": { "b": { "x": 1 }, "c": 2 }, + "items": [1, 2], + "t": [{ "a": 1 }], + "m": { "k": { "v": 2 } }, + "l": [{ "k": { "x": 1 } }] }, "options": { "strict": false }, - "specSection": "8", - "minSpecVersion": "4.2" + "specSection": "14.4", + "minSpecVersion": "4.4" }, { - "name": "skips a line above the adopted entry depth in non-strict mode", - "input": "m[1:]{a}:\n x: 1\n y: 2", - "expected": { - "m": { - "x": { - "a": 1 - } - } + "name": "throws on a line above the adopted depth in non-strict mode", + "input": "a:\n b: 1\n c: 2", + "expected": null, + "shouldError": true, + "options": { + "strict": false }, + "specSection": "14.4", + "minSpecVersion": "4.4" + }, + { + "name": "throws on a line above the adopted entry depth in non-strict mode", + "input": "m[1:]{a}:\n x: 1\n y: 2", + "expected": null, + "shouldError": true, "options": { "strict": false }, - "specSection": "8", - "minSpecVersion": "4.2" + "specSection": "14.4", + "minSpecVersion": "4.4" }, { - "name": "skips a line above the adopted row depth in non-strict mode", + "name": "throws on a line above the adopted row depth in non-strict mode", "input": "rows[2]{a}:\n 1\n x: 9\n 2", - "expected": { - "rows": [ - { - "a": 1 - }, - { - "a": 2 - } - ] - }, + "expected": null, + "shouldError": true, "options": { "strict": false }, - "specSection": "8", - "minSpecVersion": "4.2" + "specSection": "14.4", + "minSpecVersion": "4.4" }, { - "name": "skips over-indented lines under a primitive list item and after a list scope in non-strict mode", - "input": "items[2]:\n - a\n b: 1\n - c\n x: 1\nz: 2", - "expected": { - "items": [ - "a", - "c" - ], - "z": 2 - }, + "name": "throws on an over-indented line under a primitive list item in non-strict mode", + "input": "items[2]:\n - a\n b: 1\n - c", + "expected": null, + "shouldError": true, "options": { "strict": false }, "specSection": "8", - "minSpecVersion": "4.2" + "minSpecVersion": "4.4" }, { - "name": "throws on an over-indented scalar line under a primitive list item in non-strict mode", - "input": "items[2]:\n - a\n b\n - c", + "name": "throws on a non-list-item line at item depth in non-strict mode", + "input": "items[1]:\n - a\n x: 1", "expected": null, "shouldError": true, "options": { "strict": false }, - "specSection": "8", - "note": "The non-strict skip for over-indented lines excludes scalar lines (§5.2)", - "minSpecVersion": "4.1" + "specSection": "9.4", + "minSpecVersion": "4.4" }, { - "name": "skips an over-indented line under a list-item field in non-strict mode", - "input": "items[1]:\n - a: 1\n deep: 1\n b: 2", - "expected": { - "items": [ - { - "a": 1, - "b": 2 - } - ] + "name": "throws on an over-indented line under a list-item field in non-strict mode", + "input": "items[1]:\n - a: 1\n b: 2", + "expected": null, + "shouldError": true, + "options": { + "strict": false }, + "specSection": "14.4", + "note": "The hyphen line is the list-item object's first line, so the depth-jump recovery does not apply", + "minSpecVersion": "4.4" + }, + { + "name": "throws on an over-indented line under a list-item inline array in non-strict mode", + "input": "items[1]:\n - a[2]: 1,2\n b: 2", + "expected": null, + "shouldError": true, "options": { "strict": false }, - "specSection": "8", - "minSpecVersion": "4.2" + "specSection": "14.4", + "minSpecVersion": "4.4" }, { "name": "throws on a depth jump to the first tabular row", @@ -309,6 +354,20 @@ "expected": null, "shouldError": true, "specSection": "8" + }, + { + "name": "throws on a depth jump to the first list item", + "input": "items[1]:\n - a", + "expected": null, + "shouldError": true, + "specSection": "8" + }, + { + "name": "throws on a depth jump to the first entry row", + "input": "m[1:]{v}:\n k: 1", + "expected": null, + "shouldError": true, + "specSection": "8" } ] } diff --git a/tests/fixtures/decode/numbers.json b/tests/fixtures/decode/numbers.json index f35ee55..12c2e21 100644 --- a/tests/fixtures/decode/numbers.json +++ b/tests/fixtures/decode/numbers.json @@ -29,7 +29,7 @@ "b": 0 }, "specSection": "4", - "note": "Most host environments do not distinguish -0 from 0" + "note": "-0 and 0 are equal numbers (§2)" }, { "name": "parses array with mixed numeric forms", @@ -69,8 +69,8 @@ }, { "name": "treats leading-zeros in array as strings", - "input": "nums[3]: 05,007,0123", - "expected": { "nums": ["05", "007", "0123"] }, + "input": "nums[4]: 05,007,0123,01.5", + "expected": { "nums": ["05", "007", "0123", "01.5"] }, "specSection": "4" }, { @@ -100,12 +100,14 @@ "note": "The grammar admits only an optional leading minus, so leading-plus cells stay strings" }, { - "name": "treats Infinity and NaN tokens as strings", - "input": "a: Infinity\nb: -Infinity\nc: NaN", + "name": "treats Infinity, NaN, and case variants of literals as strings", + "input": "a: Infinity\nb: -Infinity\nc: NaN\nd: True\ne: NULL", "expected": { "a": "Infinity", "b": "-Infinity", - "c": "NaN" + "c": "NaN", + "d": "True", + "e": "NULL" }, "specSection": "4" }, diff --git a/tests/fixtures/decode/objects-keyed.json b/tests/fixtures/decode/objects-keyed.json index 848030a..851db01 100644 --- a/tests/fixtures/decode/objects-keyed.json +++ b/tests/fixtures/decode/objects-keyed.json @@ -153,7 +153,7 @@ "note": "Comment removal never ends a keyed scope, and a comment line never counts as an entry" }, { - "name": "applies LWW for duplicate entry keys in non-strict mode", + "name": "applies last-write-wins for duplicate entry keys in non-strict mode", "input": "m[2:]{v}:\n a: 1\n a: 2", "expected": { "m": { @@ -163,7 +163,7 @@ "options": { "strict": false }, - "specSection": "14.3", + "specSection": "14.4", "note": "Entry keys are sibling keys of the decoded object and resolve last-write-wins" }, { @@ -183,31 +183,38 @@ "minSpecVersion": "4.1" }, { - "name": "skips an entry-depth line without a colon in non-strict mode", + "name": "throws on an entry-depth line without a colon in non-strict mode", "input": "u[2:]{x}:\n a: 1\n boom", - "expected": { - "u": { - "a": { - "x": 1 - } - } - }, + "expected": null, + "shouldError": true, "options": { "strict": false }, "specSection": "9.5", - "minSpecVersion": "4.1" + "minSpecVersion": "4.4" }, { - "name": "decodes every entry row beyond the declared count in non-strict mode", - "input": "m[1:]{v}:\n a: 1\n b: 2", + "name": "throws on an entry row with fewer cells than fields in non-strict mode", + "input": "m[1:]{a,b}:\n k: 1", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "14.1", + "minSpecVersion": "4.4" + }, + { + "name": "decodes every entry row whatever the declared count in non-strict mode", + "input": "m[1:]{v}:\n a: 1\n b: 2\nn[3:]{v}:\n c: 3", "expected": { - "m": { "a": { "v": 1 }, "b": { "v": 2 } } + "m": { "a": { "v": 1 }, "b": { "v": 2 } }, + "n": { "c": { "v": 3 } } }, "options": { "strict": false }, - "specSection": "14.1", + "specSection": "14.4", "minSpecVersion": "4.1" } ] diff --git a/tests/fixtures/decode/objects.json b/tests/fixtures/decode/objects.json index 5cf1b8a..c2d46d6 100644 --- a/tests/fixtures/decode/objects.json +++ b/tests/fixtures/decode/objects.json @@ -1,7 +1,7 @@ { "version": "4.0", "category": "decode", - "description": "Object decoding – fields, nested objects, key parsing, §6 fall-through (non-strict), and §14.3 duplicate-key LWW", + "description": "Object decoding – fields, nested objects, key parsing, and §14.3 duplicate keys", "tests": [ { "name": "parses objects with primitive values", @@ -40,7 +40,7 @@ "options": { "strict": false }, - "specSection": "14.3" + "specSection": "14.4" }, { "name": "parses quoted object value with colon", @@ -83,18 +83,11 @@ "specSection": "4" }, { - "name": "parses quoted string value that looks like true", - "input": "v: \"true\"", + "name": "parses quoted field values that look like true or an integer", + "input": "v: \"true\"\nw: \"42\"", "expected": { - "v": "true" - }, - "specSection": "4" - }, - { - "name": "parses quoted string value that looks like integer", - "input": "v: \"42\"", - "expected": { - "v": "42" + "v": "true", + "w": "42" }, "specSection": "4" }, @@ -116,22 +109,14 @@ "note": "Quoting makes the value opaque (§4), so it is never parsed as a header – the matching count is irrelevant because a quoted scalar is never a header candidate" }, { - "name": "parses unquoted value shaped like an inline array header after the key", - "input": "key: foo [2]: bar", - "expected": { - "key": "foo [2]: bar" - }, - "specSection": "5.2", - "note": "An unquoted key cannot contain a colon, so the first colon ends the key; the bracket-and-colon shape in the remainder is value text, not an array header" - }, - { - "name": "parses unquoted value shaped like a tabular array header after the key", - "input": "a: b [2]{x,y}: 1,2", + "name": "parses unquoted values shaped like inline or tabular array headers after the key", + "input": "key: foo [2]: bar\na: b [2]{x,y}: 1,2", "expected": { + "key": "foo [2]: bar", "a": "b [2]{x,y}: 1,2" }, "specSection": "5.2", - "note": "An unquoted key cannot contain a colon, so the first colon ends the key; the tabular-header shape in the remainder is value text, not an array header" + "note": "An unquoted key cannot contain a colon, so the first colon ends the key; a header shape in the remainder is value text" }, { "name": "parses quoted object value with bracket text and a later colon", @@ -174,45 +159,6 @@ }, "specSection": "7.4" }, - { - "name": "treats content between bracket segment and colon as literal key (non-strict)", - "input": "foo[1][bar]: 10\nfoo[2]extra: a,b", - "options": { - "strict": false - }, - "expected": { - "foo[1][bar]": 10, - "foo[2]extra": "a,b" - }, - "specSection": "6", - "note": "Non-whitespace content between ] and : (a second bracket group or plain text) prevents array header interpretation; non-strict fall-through produces a literal key not constrained by §7.3" - }, - { - "name": "treats bracket segment without a length as literal key (non-strict)", - "input": "key[]: 1,2", - "options": { - "strict": false - }, - "expected": { - "key[]": "1,2" - }, - "specSection": "6", - "note": "An absent length prevents header interpretation; non-strict fall-through produces a literal key" - }, - { - "name": "treats bracket content that fails the length grammar as literal key (non-strict)", - "input": "foo[bar][1]: 20\nn[-1]: 1\nk[\u0662]: a,b", - "options": { - "strict": false - }, - "expected": { - "foo[bar][1]": 20, - "n[-1]": 1, - "k[\u0662]": "a,b" - }, - "specSection": "6", - "note": "[bar], [-1] and [\u0662] (a non-ASCII digit) fail the length grammar; non-strict fall-through produces a literal key not constrained by §7.3" - }, { "name": "parses quoted key with comma", "input": "\"a,b\": 1", @@ -264,9 +210,9 @@ }, { "name": "unescapes newline and quotes in keys", - "input": "\"line\\nbreak\": 1\n\"he said \\\"hi\\\"\": 2", + "input": "\"line\\n\": 1\n\"he said \\\"hi\\\"\": 2", "expected": { - "line\nbreak": 1, + "line\n": 1, "he said \"hi\"": 2 }, "specSection": "7.1" @@ -292,20 +238,22 @@ "specSection": "8" }, { - "name": "applies LWW for nested duplicate sibling keys in non-strict mode", - "input": "outer:\n name: Ada\n name: Bob", + "name": "replaces a nested duplicate object key without merging in non-strict mode", + "input": "outer:\n a:\n x: 1\n a:\n y: 2", "expected": { "outer": { - "name": "Bob" + "a": { + "y": 2 + } } }, "options": { "strict": false }, - "specSection": "14.3" + "specSection": "14.4" }, { - "name": "applies LWW for duplicate keys within a list-item object in non-strict mode", + "name": "applies last-write-wins for duplicate keys within a list-item object in non-strict mode", "input": "items[1]:\n - id: 1\n id: 2", "expected": { "items": [ @@ -317,7 +265,7 @@ "options": { "strict": false }, - "specSection": "14.3" + "specSection": "14.4" }, { "name": "accepts unquoted key outside the encoder key pattern (hyphen)", @@ -329,10 +277,13 @@ "note": "§7.3 constrains encoder output; decoders accept any token before the first unquoted colon as a literal key" }, { - "name": "accepts unquoted key outside the encoder key pattern (leading digit)", - "input": "2key: x", + "name": "accepts unquoted keys with a leading digit or spelled like a literal or number as string keys", + "input": "2key: x\ntrue: 1\nnull: 2\n1.50: 3", "expected": { - "2key": "x" + "2key": "x", + "true": 1, + "null": 2, + "1.50": 3 }, "specSection": "7.4" }, @@ -347,11 +298,12 @@ }, { "name": "accepts unquoted values and keys an encoder would quote in strict mode", - "input": "k: b:c\npaths[2]: C:\\,D:\\\na\\: b", + "input": "k: b:c\npaths[2]: C:\\,D:\\\na\\: b\ne: {}", "expected": { "k": "b:c", "paths": ["C:\\", "D:\\"], - "a\\": "b" + "a\\": "b", + "e": "{}" }, "specSection": "7.4", "note": "The first unquoted colon ends the key; the entire post-colon token is the value (§11.2)" @@ -404,16 +356,29 @@ "specSection": "15" }, { - "name": "falls through to a key-value line when whitespace precedes the bracket segment (non-strict)", - "input": "foo [2]: bar,baz", + "name": "reads an NBSP before a quote, bracket segment, or nested field group as part of the name", + "input": "\u00a0[1]: z\n\u00a0\"q\"[1]: w\nn\u00a0[1]: y\na[1]{b\u00a0{c}}:\n 1", "expected": { - "foo [2]": "bar,baz" - }, - "options": { - "strict": false + "\u00a0": [ + "z" + ], + "\u00a0\"q\"": [ + "w" + ], + "n\u00a0": [ + "y" + ], + "a": [ + { + "b\u00a0": { + "c": 1 + } + } + ] }, - "specSection": "5.2", - "minSpecVersion": "4.1" + "specSection": "6", + "note": "Whitespace is SP and HTAB only (§1.2); a leading NBSP keeps a line from being a keyless root header or a quoted key", + "minSpecVersion": "4.4" }, { "name": "accepts a header key outside the encoder unquoted-key pattern", @@ -469,12 +434,14 @@ "specSection": "15" }, { - "name": "parses an unquoted key followed by spaces before the colon", - "input": "a : 1", + "name": "parses a key-value colon with spaces before it and a key-value or header colon with none after it", + "input": "a : 1\nb:2\nc[2]:x,y", "expected": { - "a": 1 + "a": 1, + "b": 2, + "c": ["x", "y"] }, - "specSection": "7.4" + "specSection": "12" }, { "name": "parses a quoted key followed by spaces before the colon", @@ -545,18 +512,6 @@ }, "specSection": "5.2", "minSpecVersion": "4.2" - }, - { - "name": "falls through to a key-value line in non-strict mode when a header line fails the grammar", - "input": "a[2:]{x}", - "expected": { - "a[2": "]{x}" - }, - "options": { - "strict": false - }, - "specSection": "5.2", - "minSpecVersion": "4.2" } ] } diff --git a/tests/fixtures/decode/primitives.json b/tests/fixtures/decode/primitives.json index 1be157c..b2053e7 100644 --- a/tests/fixtures/decode/primitives.json +++ b/tests/fixtures/decode/primitives.json @@ -7,7 +7,7 @@ "name": "parses safe unquoted string", "input": "hello", "expected": "hello", - "specSection": "7.4" + "specSection": "4" }, { "name": "parses empty quoted string", @@ -33,6 +33,12 @@ "expected": "say \"hello\"", "specSection": "7.1" }, + { + "name": "closes a quoted token after an escaped backslash", + "input": "\"a\\\\\"[2]: \"y\\\\\",z", + "expected": { "a\\": ["y\\", "z"] }, + "specSection": "5.2" + }, { "name": "decodes \\uXXXX escape (U+0004)", "input": "val: \"a\\u0004b\"", diff --git a/tests/fixtures/decode/root-form.json b/tests/fixtures/decode/root-form.json index 10abbab..017ab9b 100644 --- a/tests/fixtures/decode/root-form.json +++ b/tests/fixtures/decode/root-form.json @@ -62,27 +62,26 @@ "specSection": "5" }, { - "name": "throws on a trailing bare token after a root array in non-strict mode", - "input": "[2]: 1,2\njunk", + "name": "throws on trailing content after a root array in non-strict mode", + "input": "[2]: 1,2\njunk: 3", "expected": null, "shouldError": true, "options": { "strict": false }, "specSection": "5", - "note": "The non-strict leniency for trailing content excludes scalar lines (§5.2)", - "minSpecVersion": "4.1" + "minSpecVersion": "4.4" }, { - "name": "throws on a bare token line under a trailing header after a root array in non-strict mode", - "input": "[1]: 1\nx[1]:\n junk", + "name": "throws on trailing content after a root empty array in non-strict mode", + "input": "[]\njunk: 3", "expected": null, "shouldError": true, "options": { "strict": false }, "specSection": "5", - "minSpecVersion": "4.1" + "minSpecVersion": "4.4" }, { "name": "parses a root line shaped like a header without a colon as a string", @@ -100,7 +99,6 @@ "strict": false }, "specSection": "5", - "note": "The non-strict skip for over-indented lines excludes scalar lines (§5.2)", "minSpecVersion": "4.2" }, { @@ -123,15 +121,15 @@ "minSpecVersion": "4.2" }, { - "name": "throws on a scalar line after a skipped over-indented first line in non-strict mode", - "input": " a: 1\nhello", + "name": "throws on a whole document indented by one level in non-strict mode", + "input": " a: 1\n b: 2", "expected": null, "shouldError": true, "options": { "strict": false }, - "specSection": "5", - "minSpecVersion": "4.2" + "specSection": "8", + "minSpecVersion": "4.4" } ] } diff --git a/tests/fixtures/decode/validation-errors.json b/tests/fixtures/decode/validation-errors.json index 56a9a62..ee03838 100644 --- a/tests/fixtures/decode/validation-errors.json +++ b/tests/fixtures/decode/validation-errors.json @@ -33,7 +33,7 @@ }, { "name": "throws on invalid escape sequence", - "input": "\"a\\x\"", + "input": "\"a\\b\"", "expected": null, "shouldError": true, "specSection": "7.1" @@ -67,7 +67,7 @@ "specSection": "14.2" }, { - "name": "throws on two primitives at root depth in strict mode", + "name": "throws on two primitives at root depth", "input": "hello\nworld", "expected": null, "shouldError": true, @@ -85,7 +85,7 @@ "note": "Active delimiter is tab; rows using comma each parse as 1 value, failing the row width check" }, { - "name": "throws on extra brackets between bracket segment and colon in strict mode", + "name": "throws on extra brackets between bracket segment and colon", "input": "foo[1][bar]: 10", "expected": null, "shouldError": true, @@ -93,10 +93,10 @@ "strict": true }, "specSection": "6", - "note": "Non-whitespace content between ] and : must error in strict mode; the fall-through is non-strict only" + "note": "The line is an array-header line that fails the §6 grammar (§5.2)" }, { - "name": "throws on text between bracket segment and colon in strict mode", + "name": "throws on text between bracket segment and colon", "input": "foo[2]extra: a,b", "expected": null, "shouldError": true, @@ -106,7 +106,7 @@ "specSection": "6" }, { - "name": "throws on non-integer bracket segment in strict mode", + "name": "throws on non-integer bracket segment", "input": "foo[bar]: 10", "expected": null, "shouldError": true, @@ -116,8 +116,8 @@ "specSection": "6" }, { - "name": "throws on duplicate sibling keys in strict mode", - "input": "name: Ada\nname: Bob", + "name": "throws on duplicate sibling keys, one of them quoted, in strict mode", + "input": "name: Ada\n\"name\": Bob", "expected": null, "shouldError": true, "options": { @@ -148,6 +148,17 @@ "specSection": "5.2", "minSpecVersion": "4.3" }, + { + "name": "throws on a keyed header line without its colon in non-strict mode", + "input": "a[2:]{x}", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "5.2", + "minSpecVersion": "4.4" + }, { "name": "throws when inline values fall short of the declared length", "input": "tags[3]: a,b", @@ -170,7 +181,7 @@ "specSection": "6" }, { - "name": "throws on bracket length with leading zeros in strict mode", + "name": "throws on bracket length with leading zeros", "input": "items[03]: a,b,c", "expected": null, "shouldError": true, @@ -181,7 +192,7 @@ "note": "[03] is not a canonical non-negative integer length; decoders MUST NOT interpret it as a bracket segment" }, { - "name": "throws on negative bracket length in strict mode", + "name": "throws on negative bracket length", "input": "items[-1]: a,b,c", "expected": null, "shouldError": true, @@ -192,7 +203,7 @@ "note": "[-1] is not a non-negative integer length; decoders MUST NOT interpret it as a bracket segment" }, { - "name": "throws on decimal bracket length in strict mode", + "name": "throws on decimal bracket length", "input": "x[3.7]: a,b,c", "expected": null, "shouldError": true, @@ -203,7 +214,7 @@ "note": "[3.7] is not an integer length – lengths are DIGIT-only – so decoders MUST NOT interpret it as a bracket segment" }, { - "name": "throws on bracket length with plus sign in strict mode", + "name": "throws on bracket length with plus sign", "input": "x[+3]: a,b,c", "expected": null, "shouldError": true, @@ -214,7 +225,7 @@ "note": "[+3] carries a sign and is not a non-negative integer length; decoders MUST NOT interpret it as a bracket segment" }, { - "name": "throws on bracket length in exponent form in strict mode", + "name": "throws on bracket length in exponent form", "input": "x[1e1]: 1,2,3,4,5,6,7,8,9,10", "expected": null, "shouldError": true, @@ -225,7 +236,15 @@ "note": "[1e1] is exponent notation, not a DIGIT-only integer length; decoders MUST NOT interpret it as a bracket segment" }, { - "name": "throws on whitespace between bracket segment and colon in strict mode", + "name": "throws on a bracket length with a non-ASCII digit", + "input": "k[\u0662]: a,b", + "expected": null, + "shouldError": true, + "specSection": "6", + "note": "A length takes ASCII digits only; a Unicode-aware \\d would accept \u0662" + }, + { + "name": "throws on whitespace between bracket segment and colon", "input": "items[2] : 1,2", "expected": null, "shouldError": true, @@ -233,10 +252,10 @@ "strict": true }, "specSection": "6", - "note": "No whitespace is permitted between ] and the colon/field list; any content there prevents header interpretation" + "note": "The line is an array-header line that fails the §6 grammar (§5.2)" }, { - "name": "throws on whitespace between bracket segment and field list in strict mode", + "name": "throws on whitespace between bracket segment and field list", "input": "items[0] {a,b}:", "expected": null, "shouldError": true, @@ -246,6 +265,13 @@ "specSection": "6", "note": "No whitespace is permitted between ] and the field list; mirrors the ]-to-colon rule" }, + { + "name": "throws on whitespace between a field list and the header colon", + "input": "a[1]{x} :\n 1", + "expected": null, + "shouldError": true, + "specSection": "6" + }, { "name": "throws on nested duplicate sibling keys in strict mode", "input": "outer:\n name: Ada\n name: Bob", @@ -283,14 +309,14 @@ "note": "The header declares three leaf fields (id, customer.name, customer.country); the row has two cells" }, { - "name": "throws on empty field list in strict mode", + "name": "throws on empty field list", "input": "items[0]{}:", "expected": null, "shouldError": true, "specSection": "14.2" }, { - "name": "throws on empty nested field group in strict mode", + "name": "throws on empty nested field group", "input": "items[0]{id,meta{}}:", "expected": null, "shouldError": true, @@ -298,12 +324,19 @@ "note": "A field list must contain at least one field entry at every nesting level" }, { - "name": "throws on unmatched brace in field list in strict mode", + "name": "throws on unmatched brace in field list", "input": "items[0]{id,customer{name:", "expected": null, "shouldError": true, "specSection": "14.2" }, + { + "name": "throws on an unmatched closing brace after a field list", + "input": "a[0]{x}}:", + "expected": null, + "shouldError": true, + "specSection": "6" + }, { "name": "throws on duplicate field names at the same field-list level in strict mode", "input": "items[0]{a,a{x}}:", @@ -312,7 +345,9 @@ "options": { "strict": true }, - "specSection": "9.3" + "specSection": "9.3", + "note": "A duplicate field name is a header defect, diagnosed from the header line alone", + "minSpecVersion": "4.1" }, { "name": "throws on entry row count mismatch with keyed header length", @@ -337,7 +372,7 @@ "note": "A bare entrykey: has zero cells, and a field list always declares at least one leaf field (§9.5)" }, { - "name": "throws on keyed header without a field list in strict mode", + "name": "throws on keyed header without a field list", "input": "m[0:]:", "expected": null, "shouldError": true, @@ -348,7 +383,7 @@ "note": "A keyed header without a field list declares no cells at all" }, { - "name": "throws on keyed marker after the delimiter symbol in strict mode", + "name": "throws on keyed marker after the delimiter symbol", "input": "m[2|:]{v}:\n a: 1\n b: 2", "expected": null, "shouldError": true, @@ -359,7 +394,7 @@ "note": "The keyed colon must immediately follow the length and precede the delimiter symbol: [2:|]" }, { - "name": "throws on keyed marker with leading-zero length in strict mode", + "name": "throws on keyed marker with leading-zero length", "input": "m[02:]{v}:\n a: 1\n b: 2", "expected": null, "shouldError": true, @@ -369,7 +404,7 @@ "specSection": "6" }, { - "name": "throws on whitespace before the keyed marker in strict mode", + "name": "throws on whitespace before the keyed marker", "input": "m[2 :]{v}:\n a: 1\n b: 2", "expected": null, "shouldError": true, @@ -379,7 +414,7 @@ "specSection": "6" }, { - "name": "throws on explicit comma delimiter after the keyed marker in strict mode", + "name": "throws on explicit comma delimiter after the keyed marker", "input": "m[2:,]{v}:\n a: 1\n b: 2", "expected": null, "shouldError": true, @@ -390,7 +425,7 @@ "note": "Comma is never written as a delimiter symbol; [N:] already declares it" }, { - "name": "throws on inline content after a keyed header colon in strict mode", + "name": "throws on inline content after a keyed header colon", "input": "m[1:]{v}: x\n a: 1", "expected": null, "shouldError": true, @@ -401,7 +436,7 @@ "note": "A keyed header carries no inline entries" }, { - "name": "throws on a line without an unquoted colon at entry depth in strict mode", + "name": "throws on a line without an unquoted colon at entry depth", "input": "m[1:]{v}:\n a: 1\n 5", "expected": null, "shouldError": true, @@ -421,7 +456,14 @@ "specSection": "14.3" }, { - "name": "throws on a keyless keyed header as a list item in strict mode", + "name": "throws on a list-item first-field header key repeated by a later header", + "input": "items[1]:\n - t[1]{a}:\n 1\n t[1]: 2", + "expected": null, + "shouldError": true, + "specSection": "14.3" + }, + { + "name": "throws on a keyless keyed header as a list item", "input": "items[1]:\n - [2:]{v}:\n a: 1\n b: 2", "expected": null, "shouldError": true, @@ -446,16 +488,15 @@ "specSection": "6" }, { - "name": "throws on a keyless list-form header in object field position in non-strict mode", - "input": "a:\n [2]:\n - 1\n - 2", + "name": "throws on keyless array header in object field position in non-strict mode", + "input": "a:\n [2]: 1,2", "expected": null, "shouldError": true, "options": { "strict": false }, "specSection": "6", - "note": "Read as a key-value line, the header's list items become scalar lines (§5.2), so no non-strict leniency yields a value", - "minSpecVersion": "4.2" + "minSpecVersion": "4.4" }, { "name": "throws on keyless array header after a depth-0 field", @@ -465,6 +506,13 @@ "specSection": "6", "note": "Keyless headers are valid only as the document's root header or as list items" }, + { + "name": "throws on keyless array header as a later field of a list-item object", + "input": "a[1]:\n - b: 1\n [2]: x,y", + "expected": null, + "shouldError": true, + "specSection": "6" + }, { "name": "throws on keyless fields-bearing header as list item", "input": "items[1]:\n - [0]{x}:", @@ -474,14 +522,14 @@ }, { "name": "throws on keyless fields-bearing header as list item in non-strict mode", - "input": "items[1]:\n - [1]{x}:\n 1", + "input": "items[1]:\n - [0]{x}:", "expected": null, "shouldError": true, "options": { "strict": false }, "specSection": "6", - "minSpecVersion": "4.2" + "minSpecVersion": "4.4" }, { "name": "throws on inline content after tabular header", @@ -492,16 +540,26 @@ "note": "A fields-bearing header carries no inline content; decoding the values as an inline array would silently drop the field list" }, { - "name": "throws on a row under a fields-bearing header with inline content in non-strict mode", - "input": "b[1]{c}: x\n y", + "name": "throws on a fields-bearing header with inline content in non-strict mode", + "input": "b[1]{c}: x", "expected": null, "shouldError": true, "options": { "strict": false }, "specSection": "6", - "note": "Read as a key-value line, the row becomes an over-indented scalar line (§5.2), so no non-strict leniency yields a value", - "minSpecVersion": "4.2" + "minSpecVersion": "4.4" + }, + { + "name": "throws on a fields-bearing header with inline content after a list-item hyphen in non-strict mode", + "input": "items[1]:\n - k[1]{a}: 1", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "6", + "minSpecVersion": "4.4" }, { "name": "throws on inline content after root tabular header", @@ -510,6 +568,14 @@ "shouldError": true, "specSection": "6" }, + { + "name": "throws on a tab after a fields-bearing header's colon", + "input": "a[1:]{x}:\t\n k: 1", + "expected": null, + "shouldError": true, + "specSection": "6", + "minSpecVersion": "4.4" + }, { "name": "throws on whitespace between a key and its bracket segment", "input": "foo [2]: bar,baz", @@ -519,12 +585,30 @@ "minSpecVersion": "4.1" }, { - "name": "throws on duplicate field names in a zero-row tabular header", - "input": "items[0]{a,a}:", + "name": "throws on a space before the bracket segment in non-strict mode", + "input": "foo [2]: bar,baz", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "6", + "minSpecVersion": "4.4" + }, + { + "name": "throws on a tab between a key and its bracket segment", + "input": "t\t[1]: x", + "expected": null, + "shouldError": true, + "specSection": "6", + "minSpecVersion": "4.1" + }, + { + "name": "throws on duplicate field names inside a nested field group", + "input": "items[0]{n{x,x}}:", "expected": null, "shouldError": true, "specSection": "9.3", - "note": "A duplicate field name is a header defect, diagnosed from the header line alone", "minSpecVersion": "4.1" }, { @@ -563,7 +647,7 @@ "minSpecVersion": "4.1" }, { - "name": "throws on the removed [#N] length marker in a root header in strict mode", + "name": "throws on the removed [#N] length marker in a root header", "input": "[#2]: a,b", "expected": null, "shouldError": true, @@ -573,7 +657,7 @@ "specSection": "6" }, { - "name": "throws on the removed [#N] length marker in a field header in strict mode", + "name": "throws on the removed [#N] length marker in a field header", "input": "xs[#2]: a,b", "expected": null, "shouldError": true, @@ -583,7 +667,7 @@ "specSection": "6" }, { - "name": "throws on the removed [#N] length marker in a keyless list-item header in strict mode", + "name": "throws on the removed [#N] length marker in a keyless list-item header", "input": "items[1]:\n - [#2]: a,b", "expected": null, "shouldError": true, @@ -593,7 +677,7 @@ "specSection": "6" }, { - "name": "throws on the removed [#N] length marker in a list-item first-field header in strict mode", + "name": "throws on the removed [#N] length marker in a list-item first-field header", "input": "items[1]:\n - xs[#2]: a,b", "expected": null, "shouldError": true, @@ -652,8 +736,8 @@ "note": "The colon precedes the delimiter, so the line ends the rows instead of becoming a second row" }, { - "name": "throws on a header delimiter mismatch that row widths do not expose", - "input": "items[1|]{a,b}:\n 1,2", + "name": "throws on a header delimiter mismatch inside a nested field group that row widths do not expose", + "input": "items[1|]{a|n{x,y}}:\n 1|2", "expected": null, "shouldError": true, "options": { @@ -661,6 +745,28 @@ }, "specSection": "6" }, + { + "name": "throws on a tab field list after a pipe delimiter symbol in non-strict mode", + "input": "a[0|]{x\ty}:", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "6", + "minSpecVersion": "4.4" + }, + { + "name": "throws on a pipe field list after a bracket segment without a delimiter symbol in non-strict mode", + "input": "a[0]{x|y}:", + "expected": null, + "shouldError": true, + "options": { + "strict": false + }, + "specSection": "6", + "minSpecVersion": "4.4" + }, { "name": "throws on a hyphen without a following space at item depth", "input": "items[2]:\n - a\n -b", @@ -725,15 +831,14 @@ }, { "name": "throws on content after a nested field group in non-strict mode", - "input": "a[1]{b{c}d}:\n 1", + "input": "a[0]{b{c}d}:", "expected": null, "shouldError": true, "options": { "strict": false }, "specSection": "6", - "note": "Not a header (§6 grammar), so the row is a scalar line under a key-value line (§5.2)", - "minSpecVersion": "4.2" + "minSpecVersion": "4.4" }, { "name": "throws on a \\u escape with a sign before its hex digits", diff --git a/tests/fixtures/decode/whitespace.json b/tests/fixtures/decode/whitespace.json index 3585de6..028e0fd 100644 --- a/tests/fixtures/decode/whitespace.json +++ b/tests/fixtures/decode/whitespace.json @@ -34,11 +34,11 @@ }, { "name": "parses empty tokens as empty string", - "input": "items[3]: a,,c", + "input": "items[4]: a,,c,", "expected": { - "items": ["a", "", "c"] + "items": ["a", "", "c", ""] }, - "specSection": "9.1" + "specSection": "11.2" }, { "name": "keeps a leading NBSP and a trailing tab in unquoted values", @@ -59,6 +59,17 @@ "specSection": "12", "note": "Token trimming removes U+0020 only; a host trim() that also strips NBSP or tab corrupts this round-trip" }, + { + "name": "keeps an NBSP or a tab around a key or field name", + "input": "a\u00a0: 1\nb\t: 2\nt[1]{\u00a0c}:\n 3", + "expected": { + "a\u00a0": 1, + "b\t": 2, + "t": [{ "\u00a0c": 3 }] + }, + "specSection": "12", + "minSpecVersion": "4.1" + }, { "name": "decodes CRLF line terminators", "input": "a: 1\r\nb: 2\r\n", @@ -71,13 +82,6 @@ "expected": { "items": [{ "a": 1, "b": 2 }, { "a": 3, "b": 4 }] }, "specSection": "12" }, - { - "name": "keeps an escaped carriage return inside a quoted value", - "input": "a: \"x\\ry\"\r\n", - "expected": { "a": "x\ry" }, - "specSection": "12", - "note": "Only a CR at the end of a line is part of the line terminator" - }, { "name": "treats a carriage-return-only line as blank", "input": "a: 1\n\r\nb: 2", @@ -101,7 +105,7 @@ }, { "name": "treats a hyphen followed by trailing spaces as the bare marker", - "input": "a[1]:\n - ", + "input": "a[1]: \n - ", "expected": { "a": [ {} @@ -111,6 +115,33 @@ "note": "Trailing spaces are stripped before line classification", "minSpecVersion": "4.1" }, + { + "name": "tolerates extra spaces after a list-item hyphen and keeps an NBSP after them", + "input": "items[3]:\n - \u00a0a\n - [2]: 1,2\n - k[1]: v", + "expected": { + "items": [ + "\u00a0a", + [ + 1, + 2 + ], + { + "k": [ + "v" + ] + } + ] + }, + "specSection": "5.2", + "minSpecVersion": "4.4" + }, + { + "name": "throws on a tab after a list-item hyphen", + "input": "items[1]:\n -\tx", + "expected": null, + "shouldError": true, + "specSection": "5.2" + }, { "name": "keeps a second carriage return before CRLF as content", "input": "a: x\r\r\nb: 1", @@ -121,10 +152,11 @@ "specSection": "12" }, { - "name": "keeps a second leading byte-order mark as content", - "input": "\ufeff\ufeffa: 1", + "name": "keeps a byte-order mark after the document start as content", + "input": "\ufeff\ufeffa: 1\n\ufeffb: 2", "expected": { - "\ufeffa": 1 + "\ufeffa": 1, + "\ufeffb": 2 }, "specSection": "12", "minSpecVersion": "4.1" diff --git a/tests/fixtures/encode/arrays-nested.json b/tests/fixtures/encode/arrays-nested.json index 079ddfb..984f637 100644 --- a/tests/fixtures/encode/arrays-nested.json +++ b/tests/fixtures/encode/arrays-nested.json @@ -86,8 +86,7 @@ }, "expected": "a[1]:\n - [2]:\n - x: 1\n - x: 2", "specSection": "9.4", - "note": "A keyless fields-bearing header is valid only at the document root", - "minSpecVersion": "4.1" + "note": "A keyless fields-bearing header is valid only at the document root" }, { "name": "uses list form for an array of arrays in list-item position", diff --git a/tests/fixtures/encode/arrays-primitive.json b/tests/fixtures/encode/arrays-primitive.json index 5a2ab0e..06d3678 100644 --- a/tests/fixtures/encode/arrays-primitive.json +++ b/tests/fixtures/encode/arrays-primitive.json @@ -78,9 +78,9 @@ { "name": "quotes strings with structural meanings in arrays", "input": { - "items": ["[5]", "- item", "{key}"] + "items": ["[5", "x]", "- item", "{key", "y}"] }, - "expected": "items[3]: \"[5]\",\"- item\",\"{key}\"", + "expected": "items[5]: \"[5\",\"x]\",\"- item\",\"{key\",\"y}\"", "specSection": "9.1" }, { diff --git a/tests/fixtures/encode/objects.json b/tests/fixtures/encode/objects.json index ad6271f..1af7199 100644 --- a/tests/fixtures/encode/objects.json +++ b/tests/fixtures/encode/objects.json @@ -53,11 +53,12 @@ "specSection": "7.2" }, { - "name": "quotes string value with leading space", + "name": "quotes string values with a leading or trailing space", "input": { - "text": " padded " + "lead": " padded", + "trail": "padded " }, - "expected": "text: \" padded \"", + "expected": "lead: \" padded\"\ntrail: \"padded \"", "specSection": "7.2" }, { @@ -161,21 +162,22 @@ "note": "The unquoted-key pattern is ASCII-only, so every non-ASCII key is quoted" }, { - "name": "keeps a dotted key unquoted", + "name": "keeps dotted and literal-named keys unquoted", "input": { - "a.b": 1 + "a.b": 1, + "true": 2 }, - "expected": "a.b: 1", + "expected": "a.b: 1\ntrue: 2", "specSection": "7.3", "minSpecVersion": "4.2" }, { "name": "escapes newline and quotes in keys", "input": { - "line\nbreak": 1, + "line\n": 1, "he said \"hi\"": 2 }, - "expected": "\"line\\nbreak\": 1\n\"he said \\\"hi\\\"\": 2", + "expected": "\"line\\n\": 1\n\"he said \\\"hi\\\"\": 2", "specSection": "7.1" }, { @@ -274,9 +276,10 @@ "b": "NaN", "c": "True", "d": "\ufeffx", - "e": "a\u007fb" + "e": "a\u007fb", + "f": "\u0664\u0662" }, - "expected": "a: .5\nb: NaN\nc: True\nd: \ufeffx\ne: a\u007fb", + "expected": "a: .5\nb: NaN\nc: True\nd: \ufeffx\ne: a\u007fb\nf: \u0664\u0662", "specSection": "7.2", "note": "Only a root primitive starting with U+FEFF is quoted, and only U+0000–U+001F count as controls (§7.1, §7.2)", "minSpecVersion": "4.2" diff --git a/tests/fixtures/encode/primitives.json b/tests/fixtures/encode/primitives.json index 1891165..bfb12ad 100644 --- a/tests/fixtures/encode/primitives.json +++ b/tests/fixtures/encode/primitives.json @@ -71,22 +71,22 @@ "specSection": "7.1" }, { - "name": "escapes U+0004 via \\uXXXX and keeps U+2028 literal in quoted strings", - "input": { "val": "a\u0004b", "sep": "a:\u2028b" }, - "expected": "val: \"a\\u0004b\"\nsep: \"a:\u2028b\"", + "name": "escapes U+0004, U+0008, and U+000C via \\uXXXX and keeps DEL, U+0085, and U+2028 literal in quoted strings", + "input": { "val": "a\u0004\b\fb", "sep": "a:\u007f\u0085\u2028b" }, + "expected": "val: \"a\\u0004\\u0008\\u000cb\"\nsep: \"a:\u007f\u0085\u2028b\"", "specSection": "7.1", "minSpecVersion": "4.2" }, { "name": "escapes backslash in string", - "input": "C:\\Users\\path", - "expected": "\"C:\\\\Users\\\\path\"", + "input": "Users\\path", + "expected": "\"Users\\\\path\"", "specSection": "7.1" }, { - "name": "keeps a string with an interior hash unquoted", - "input": "a #b", - "expected": "a #b", + "name": "keeps a string with an interior hyphen or hash unquoted", + "input": "a-b #c", + "expected": "a-b #c", "specSection": "7.2", "minSpecVersion": "4.2" }, @@ -111,8 +111,8 @@ }, { "name": "quotes string with bracket notation", - "input": "[test]", - "expected": "\"[test]\"", + "input": "a[test]", + "expected": "\"a[test]\"", "specSection": "7.2" }, { @@ -200,7 +200,8 @@ "name": "encodes floats with their shortest round-trip digits", "input": [0.3333333333333333, 0.30000000000000004, -0.000001], "expected": "[3]: 0.3333333333333333,0.30000000000000004,-0.000001", - "specSection": "2" + "specSection": "2", + "minSpecVersion": "4.2" }, { "name": "encodes true", @@ -227,18 +228,12 @@ "specSection": "7.2", "note": "Unquoted, a v3 host-parser decoder would read the number 1" }, - { - "name": "quotes string equal to hash", - "input": "#", - "expected": "\"#\"", - "specSection": "7.2", - "note": "Comment marker at position 0 must be quoted" - }, { "name": "quotes string starting with hash", "input": "#hello", "expected": "\"#hello\"", - "specSection": "7.2" + "specSection": "7.2", + "note": "Comment marker at position 0 must be quoted" }, { "name": "quotes a root string starting with U+FEFF", @@ -255,6 +250,13 @@ "specSection": "2", "note": "The binary64 value is exactly 123456789012345683968", "minSpecVersion": "4.2" + }, + { + "name": "quotes a delimiter, colon, hyphen, or number sign that a combining mark follows", + "input": { "items": ["a,\u0301b", "-\u0301", "#\u0301"], "k": "x:\u0301" }, + "expected": "items[3]: \"a,\u0301b\",\"-\u0301\",\"#\u0301\"\nk: \"x:\u0301\"", + "specSection": "1.2", + "minSpecVersion": "4.4" } ] } diff --git a/tests/fixtures/encode/whitespace.json b/tests/fixtures/encode/whitespace.json index 07dda0a..22c345f 100644 --- a/tests/fixtures/encode/whitespace.json +++ b/tests/fixtures/encode/whitespace.json @@ -37,7 +37,7 @@ "expected": "k: \u00a0x\u00a0", "specSection": "7.2", "note": "The whitespace trigger is U+0020 and U+0009 only", - "minSpecVersion": "4.1" + "minSpecVersion": "4.2" }, { "name": "indents list-item sibling fields by indentSize", From f114a43857c331e37f36787aac752ba467e6f8a6 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Thu, 8 Oct 2026 17:51:36 +0200 Subject: [PATCH 12/13] feat: target TOON spec 4.4 --- AGENTS.md | 2 +- CHANGELOG.md | 6 +++--- README.md | 6 +++--- docs/data-types.md | 2 +- docs/index.md | 2 +- src/toon_format/__init__.py | 2 +- tests/test_api.py | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d54ba11..c02c34c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ and mirrors the `json` module: `dumps`, `dump`, `loads`, `load`, and `ToonDecodeError`, with `encode` and `decode` as aliases of `dumps` and `loads`, the names the other TOON implementations use. -**Target specification: TOON 4.3**, exposed as `toon_format.__toon_spec__`. +**Target specification: TOON 4.4**, exposed as `toon_format.__toon_spec__`. The conformance fixtures in `tests/fixtures/` are copied from the spec repository and never edited by hand. The specification is the authority: when this code, its tests, or another implementation disagree with diff --git a/CHANGELOG.md b/CHANGELOG.md index c25886d..cd08a2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,9 @@ All notable changes to this project are documented here. The format follows ### Changed -- Support for [TOON specification 4.3](https://github.com/toon-format/spec/blob/v4.3.0/SPEC.md): - all official fixtures of spec `v4.3.0` pass, and `toon_format.__toon_spec__` - is `"4.3"`. Encoder output is unchanged. +- Support for [TOON specification 4.4](https://github.com/toon-format/spec/blob/v4.4.0/SPEC.md): + all official fixtures of spec `v4.4.0` pass, and `toon_format.__toon_spec__` + is `"4.4"`. Encoder output is unchanged. - A line whose first unquoted `[` precedes its first unquoted colon and that fails the header grammar is an error, also when its only colon sits in the bracket segment or field list (`a[1:`, `a[2:]{x}`). diff --git a/README.md b/README.md index b9b97b2..e43e1f1 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![CI](https://github.com/toon-format/toon-python/actions/workflows/ci.yml/badge.svg)](https://github.com/toon-format/toon-python/actions/workflows/ci.yml) [![PyPI](https://img.shields.io/pypi/v/toon-format.svg)](https://pypi.org/project/toon-format/) [![Python versions](https://img.shields.io/pypi/pyversions/toon-format.svg)](https://pypi.org/project/toon-format/) -[![TOON spec](https://img.shields.io/badge/TOON%20spec-4.3-blue)](https://github.com/toon-format/spec/blob/main/SPEC.md) +[![TOON spec](https://img.shields.io/badge/TOON%20spec-4.4-blue)](https://github.com/toon-format/spec/blob/main/SPEC.md) [![Documentation](https://readthedocs.org/projects/toon-format/badge/?version=stable)](https://toon-format.readthedocs.io/en/stable/) The official Python implementation of **TOON** (Token-Oriented Object Notation): @@ -11,7 +11,7 @@ a compact, human-readable encoding of the JSON data model, designed to save tokens in LLM prompts. Keep working with dictionaries and lists in your code, and encode them as TOON only where a model reads them. -- **Complete**: implements [TOON specification 4.3](https://github.com/toon-format/spec/blob/main/SPEC.md) +- **Complete**: implements [TOON specification 4.4](https://github.com/toon-format/spec/blob/main/SPEC.md) and passes every official conformance fixture. - **Familiar**: `dumps`, `dump`, `loads`, and `load`, like the `json` module, plus [`encode` and `decode`](#encode-and-decode), the names of the @@ -174,7 +174,7 @@ implementations, and Pydantic. The distribution and the module keep their names, and `encode(value)` and `decode(text)` keep working. Options are now keyword arguments instead of a dictionary, the token helpers and the `toon` command are gone, and the output -follows specification 4.3. See the +follows specification 4.4. See the [migration guide](https://toon-format.readthedocs.io/en/stable/migration/). ## Contributing diff --git a/docs/data-types.md b/docs/data-types.md index 0f5c984..cd6bb51 100644 --- a/docs/data-types.md +++ b/docs/data-types.md @@ -30,7 +30,7 @@ converted back to dates or other types. ## Specification choices -This package targets **TOON 4.3** (`toon_format.__toon_spec__`). The specification +This package targets **TOON 4.4** (`toon_format.__toon_spec__`). The specification asks implementations to document the following choices. **Numbers.** Integers encode and decode exactly at any size. Floats use the diff --git a/docs/index.md b/docs/index.md index b4a247f..19b2d84 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ The official Python implementation of **TOON** (Token-Oriented Object Notation): a compact, human-readable encoding of the JSON data model, designed to save tokens in LLM prompts. -- **Complete**: implements [TOON specification 4.3](https://github.com/toon-format/spec/blob/main/SPEC.md) +- **Complete**: implements [TOON specification 4.4](https://github.com/toon-format/spec/blob/main/SPEC.md) and passes every official conformance fixture. - **Familiar**: `dumps`, `dump`, `loads`, and `load`, like the `json` module, with `encode` and `decode` as aliases. diff --git a/src/toon_format/__init__.py b/src/toon_format/__init__.py index 880065a..b4efa9b 100644 --- a/src/toon_format/__init__.py +++ b/src/toon_format/__init__.py @@ -37,7 +37,7 @@ ToonDecodeError.__module__ = __name__ # shown as toon_format.ToonDecodeError -__toon_spec__ = "4.3" +__toon_spec__ = "4.4" """Version of the TOON specification this package implements.""" try: diff --git a/tests/test_api.py b/tests/test_api.py index 5e1eecb..c0fd0bd 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -27,7 +27,7 @@ def test_public_names() -> None: "loads", ] ) - assert toon_format.__toon_spec__ == "4.3" + assert toon_format.__toon_spec__ == "4.4" assert re.fullmatch(r"\d+\.\d+\.\d+.*", toon_format.__version__) From 8aed54f4026748ee74ec7707ac2ab0405e89e835 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Thu, 8 Oct 2026 18:25:38 +0200 Subject: [PATCH 13/13] test(decode): drop the non-strict cases the fixtures already pin --- tests/test_decode.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/tests/test_decode.py b/tests/test_decode.py index 3ff8483..ddd5275 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -171,22 +171,7 @@ def test_deep_nesting_raises_decode_error() -> None: toon_format.loads(text) -# -- non-strict leniency (§12, §14) ---------------------------------------------------- - - -@pytest.mark.parametrize( - ("text", "expected"), - [ - ("a:\n\tb: 1", {"a": {"b": 1}}), # a tab is one level - ("a:\n b: 1\n c: 2", {"a": {"b": 1, "c": 2}}), # depth jump - ("items[2]:\n - x\n - y", {"items": ["x", "y"]}), - ("a: 1\nb: 2\na: 3", {"a": 3, "b": 2}), # last write wins, first position kept - ], -) -def test_non_strict(text: str, expected: Any) -> None: - with pytest.raises(toon_format.ToonDecodeError): - toon_format.loads(text) - assert toon_format.loads(text, strict=False) == expected +# -- indentation (§12) ----------------------------------------------------------------- def test_indent_size_option() -> None: