From 98c6bc86b9c10229024cae1701caf670fab8b537 Mon Sep 17 00:00:00 2001 From: Artfizer Date: Tue, 8 Sep 2026 22:27:01 +0300 Subject: [PATCH 01/13] feat: add JSON validation CLI command (validate-json) Add a validate-json CLI command that scans JSON files, registers GTS schemas and instances, and reports validation issues for given json file or folder with *.json files Signed-off-by: Artfizer --- gts/src/gts/_cli.py | 17 +++ gts/src/gts/_json_validation.py | 244 ++++++++++++++++++++++++++++++++ tests/test_json_validation.py | 79 +++++++++++ 3 files changed, 340 insertions(+) create mode 100644 gts/src/gts/_json_validation.py create mode 100644 tests/test_json_validation.py diff --git a/gts/src/gts/_cli.py b/gts/src/gts/_cli.py index fc5cd1b..0134c4e 100644 --- a/gts/src/gts/_cli.py +++ b/gts/src/gts/_cli.py @@ -5,6 +5,7 @@ import logging import sys +from ._json_validation import GtsJsonValidator from ._server import GtsHttpServer from .ops import GtsOps @@ -34,6 +35,11 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--gts-id", required=True) s.add_argument("--scope", choices=["major", "full"], default="major") + s = sub.add_parser( + "validate-json", help="Validate all JSON documents in a file or directory" + ) + s.add_argument("--path", dest="scan_path", help="JSON file or directory to scan") + s = sub.add_parser( "validate-instance", help="Validate an instance against its schema" ) @@ -146,6 +152,17 @@ def main(argv: list[str] | None = None) -> None: json.dump(out, sys.stdout, ensure_ascii=False, indent=2) sys.stdout.write("\n") return + elif args.op == "validate-json": + scan_path = args.scan_path or args.path + if not scan_path: + parser.error("validate-json requires --path") + result = GtsJsonValidator(scan_path, ops.cfg).validate() + for issue in result.issues: + suffix = f"#{issue.index}" if issue.index is not None else "" + sys.stderr.write( + f"{issue.file}{suffix}: {issue.stage}: {issue.message}\n" + ) + out = result.to_dict() elif args.op == "validate-id": out = ops.validate_id(args.gts_id).to_dict() elif args.op == "parse-id": diff --git a/gts/src/gts/_json_validation.py b/gts/src/gts/_json_validation.py new file mode 100644 index 0000000..fa533d9 --- /dev/null +++ b/gts/src/gts/_json_validation.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +import json +import os +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from jsonschema.validators import validator_for + +from .entities import GtsEntity, GtsFile +from .gts import GtsID +from .store import GtsStore + + +@dataclass +class GtsJsonValidationIssue: + file: str + stage: str + message: str + index: int | None = None + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "file": self.file, + "stage": self.stage, + "message": self.message, + } + if self.index is not None: + result["index"] = self.index + return result + + +@dataclass +class GtsJsonValidationResult: + files: int = 0 + documents: int = 0 + gts_entities: int = 0 + schemas: int = 0 + instances: int = 0 + issues: list[GtsJsonValidationIssue] = field(default_factory=list) + + @property + def ok(self) -> bool: + return not self.issues + + def to_dict(self) -> dict[str, Any]: + return { + "ok": self.ok, + "files": self.files, + "documents": self.documents, + "gts_entities": self.gts_entities, + "schemas": self.schemas, + "instances": self.instances, + "issues": [issue.to_dict() for issue in self.issues], + } + + +class GtsJsonValidator: + def __init__(self, path: str, cfg: Any) -> None: + self.path = Path(path).expanduser() + self.cfg = cfg + self.result = GtsJsonValidationResult() + self.entities: list[GtsEntity] = [] + + def validate(self) -> GtsJsonValidationResult: + for file_path in self._json_files(): + self._read_file(file_path) + self._validate_json_schemas() + store = self._register_gts_entities() + self._validate_schemas(store) + self._validate_instances(store) + return self.result + + def _json_files(self) -> list[Path]: + resolved = self.path.resolve(strict=False) + if resolved.is_file(): + if resolved.suffix.lower() == ".json": + return [resolved] + self._issue(resolved, "discovery", "Expected a .json file") + return [] + if not resolved.is_dir(): + self._issue( + resolved, "discovery", "Path does not exist or is not accessible" + ) + return [] + + files: list[Path] = [] + for root, dirs, names in os.walk(resolved, followlinks=True): + dirs[:] = [ + name for name in dirs if name not in {"node_modules", "dist", "build"} + ] + files.extend( + Path(root, name).resolve(strict=False) + for name in names + if Path(name).suffix.lower() == ".json" + ) + return sorted(set(files)) + + def _read_file(self, file_path: Path) -> None: + self.result.files += 1 + try: + with file_path.open(encoding="utf-8") as source: + content = json.load(source) + except Exception as error: # noqa: BLE001 - report this document and continue + self._issue(file_path, "json", str(error)) + return + + values = content if isinstance(content, list) else [content] + file = GtsFile(path=str(file_path), name=file_path.name, content=content) + for index, value in enumerate(values): + self.result.documents += 1 + self.entities.append( + GtsEntity( + file=file, + list_sequence=index if isinstance(content, list) else None, + content=value, + cfg=self.cfg, + ) + ) + + def _validate_json_schemas(self) -> None: + for entity in self.entities: + content = entity.content + if not isinstance(content, dict) or "$schema" not in content: + continue + try: + validator_for(content).check_schema(content) + except Exception as error: # noqa: BLE001 - report this document and continue + self._issue(entity, "json-schema", str(error)) + + def _register_gts_entities(self) -> GtsStore: + store = GtsStore(reader=None) # type: ignore[arg-type] + keys: set[str] = set() + for entity in self.entities: + if not self._is_gts_related(entity.content): + continue + key = self._registry_key(entity) + if key is None: + self._issue( + entity, "registry", "GTS-related document has no registrable GTS ID" + ) + continue + if not entity.is_schema and entity.selected_entity_field is None: + raw_id = entity.raw_id + if raw_id is not None: + entity.raw_id = str(uuid.uuid5(uuid.NAMESPACE_URL, raw_id)) + key = entity.raw_id + if key in keys: + self._issue(entity, "registry", f"Duplicate GTS entity ID '{key}'") + continue + keys.add(key) + store.register(entity) + self.result.gts_entities += 1 + if entity.is_schema: + self.result.schemas += 1 + else: + self.result.instances += 1 + return store + + @staticmethod + def _is_gts_related(value: Any) -> bool: + if isinstance(value, str): + return "gts." in value + if isinstance(value, dict): + return any( + GtsJsonValidator._is_gts_related(item) for item in value.values() + ) + if isinstance(value, list): + return any(GtsJsonValidator._is_gts_related(item) for item in value) + return False + + @staticmethod + def _registry_key(entity: GtsEntity) -> str | None: + if entity.is_schema and entity.gts_id: + return entity.gts_id.id + if ( + not entity.is_schema + and entity.raw_id + and ( + entity.gts_id + or (entity.type_id is not None and GtsID.is_valid(entity.type_id)) + ) + ): + return entity.raw_id + return None + + def _validate_schemas(self, store: GtsStore) -> None: + schemas = sorted( + ( + entity + for entity in self.entities + if entity.is_schema + and entity.gts_id + and store.get(entity.gts_id.id) is entity + ), + key=self._schema_depth, + ) + for stage, depth in (("base-type", 1), ("derived-type", None)): + for entity in schemas: + if (depth == 1) != (self._schema_depth(entity) == 1): + continue + gts_id = entity.gts_id + if not gts_id: + continue + try: + store.validate_schema(gts_id.id) + except Exception as error: # noqa: BLE001 - report this document and continue + self._issue(entity, stage, str(error)) + + @staticmethod + def _schema_depth(entity: GtsEntity) -> int: + return len(entity.gts_id.gts_id_segments) if entity.gts_id else 0 + + def _validate_instances(self, store: GtsStore) -> None: + for entity in self.entities: + key = self._registry_key(entity) + if ( + entity.is_schema + or key is None + or not self._is_gts_related(entity.content) + ): + continue + try: + store.validate_instance(key) + except Exception as error: # noqa: BLE001 - report this document and continue + self._issue(entity, "instance", str(error)) + + def _issue( + self, + source: Path | GtsEntity, + stage: str, + message: str, + ) -> None: + if isinstance(source, GtsEntity): + file = source.file.path if source.file else source.label + index = source.list_sequence + else: + file = str(source) + index = None + self.result.issues.append( + GtsJsonValidationIssue(file=file, stage=stage, message=message, index=index) + ) diff --git a/tests/test_json_validation.py b/tests/test_json_validation.py new file mode 100644 index 0000000..7f06f81 --- /dev/null +++ b/tests/test_json_validation.py @@ -0,0 +1,79 @@ +import json + +from gts._cli import main +from gts.entities import DEFAULT_GTS_CONFIG +from gts._json_validation import GtsJsonValidator + + +def test_validate_json_reports_all_document_errors(tmp_path): + (tmp_path / "broken.json").write_text("{", encoding="utf-8") + (tmp_path / "invalid-schema.json").write_text( + json.dumps( + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "gts://gts.example.catalog._.item.v1~", + "type": 3, + } + ), + encoding="utf-8", + ) + (tmp_path / "instance.json").write_text( + json.dumps( + { + "id": "gts.example.catalog._.item.v1~example.catalog._.one.v1", + "type": "gts.example.catalog._.item.v1~", + } + ), + encoding="utf-8", + ) + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert result.files == 3 + assert result.documents == 2 + assert result.schemas == 1 + assert result.instances == 1 + assert not result.ok + assert {issue.stage for issue in result.issues} >= { + "json", + "json-schema", + "instance", + } + + +def test_validate_json_registers_type_only_instances(tmp_path): + type_id = "gts.example.catalog._.item.v1~" + (tmp_path / "schema.json").write_text( + json.dumps( + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": f"gts://{type_id}", + "type": "object", + "required": ["type", "name"], + "properties": {"type": {"const": type_id}, "name": {"type": "string"}}, + } + ), + encoding="utf-8", + ) + (tmp_path / "instance.json").write_text( + json.dumps({"type": type_id, "name": "anonymous"}), encoding="utf-8" + ) + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert result.ok + assert result.schemas == 1 + assert result.instances == 1 + + +def test_validate_json_cli_emits_issues_to_stderr(tmp_path, capsys): + input_path = tmp_path / "broken.json" + input_path.write_text("{", encoding="utf-8") + + main(["validate-json", "--path", str(input_path)]) + + captured = capsys.readouterr() + output = json.loads(captured.out) + assert output["ok"] is False + assert output["issues"][0]["file"] == str(input_path) + assert f"{input_path}: json:" in captured.err From 1712a17418cfccaa4a22f96c6b8bbe4a99345986 Mon Sep 17 00:00:00 2001 From: Artfizer Date: Tue, 8 Sep 2026 22:28:37 +0300 Subject: [PATCH 02/13] chore: bump version to 0.13.1 Signed-off-by: Artfizer --- gts/openapi.json | 2 +- gts/pyproject.toml | 2 +- gts/src/gts/_server.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gts/openapi.json b/gts/openapi.json index df00e6f..69dc264 100644 --- a/gts/openapi.json +++ b/gts/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "GTS Server", - "version": "0.13.0" + "version": "0.13.1" }, "paths": { "/entities": { diff --git a/gts/pyproject.toml b/gts/pyproject.toml index 893c599..970fb5d 100644 --- a/gts/pyproject.toml +++ b/gts/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "gts" -version = "0.13.0" +version = "0.13.1" description = "Global Type System (GTS) helpers: identifiers, parsing, validation, and operations" readme = "README.md" authors = [{ name = "GTS Community" }] diff --git a/gts/src/gts/_server.py b/gts/src/gts/_server.py index 653e65b..639ae16 100644 --- a/gts/src/gts/_server.py +++ b/gts/src/gts/_server.py @@ -185,7 +185,7 @@ def __init__( self.host = host self.port = port self.base_url = f"http://{self.host}:{self.port}" - self.app = FastAPI(title="GTS Server", version="0.13.0") + self.app = FastAPI(title="GTS Server", version="0.13.1") self.app.add_middleware( _RequestLoggingMiddleware, verbose=self.ops.verbose, From fa100659fe7cff780ddc6d4911c87b2cb6dd0d13 Mon Sep 17 00:00:00 2001 From: Artfizer Date: Tue, 8 Sep 2026 22:44:11 +0300 Subject: [PATCH 03/13] docs: add supported GTS spec version to README.md Signed-off-by: Artfizer --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 594c9f9..45272f0 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and JSON/JSON Schema artifacts. -## Roadmap +Current supported GTS spec version: `0.13.1` -Current supported GTS spec version: 0.13 +## Roadmap Featureset: From bbf99145f627dd7a5a5fbebc239a10ccaa9dca0b Mon Sep 17 00:00:00 2001 From: Artfizer Date: Thu, 10 Sep 2026 01:42:40 +0300 Subject: [PATCH 04/13] refactor(validate-all): marker heuristic, sorted errors, JSON-only output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace hardcoded directory excludes with text-based GTS marker heuristic: skip files whose raw text lacks "gts.", "gts://", or "x-gts-ref" before paying JSON parse cost. - Rename _validate_json_schemas → _check_schema_field_type (type-only). - Report malformed/non-GTS schema $id distinctly ("registry" stage). - Sort schema errors by (depth, gts_id, file, index): base-type first, then derived-type, each in total order on the remaining keys. - Sort instance errors by (depth, gts_id, file, index). - Update _is_gts_related to check gts://, x-gts-ref in addition to gts. - Remove stderr issue printing from validate-all CLI; output JSON only. - Add tests: malformed ID, incidental mention, duplicate entity, non-GTS file filtering, marker heuristic, schema/instance ordering, JSON-only CLI output. Signed-off-by: Artfizer --- gts/src/gts/_cli.py | 11 +-- gts/src/gts/_json_validation.py | 159 ++++++++++++++++++----------- tests/test_json_validation.py | 170 ++++++++++++++++++++++++++++++-- 3 files changed, 269 insertions(+), 71 deletions(-) diff --git a/gts/src/gts/_cli.py b/gts/src/gts/_cli.py index 0134c4e..398a3c4 100644 --- a/gts/src/gts/_cli.py +++ b/gts/src/gts/_cli.py @@ -36,7 +36,7 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--scope", choices=["major", "full"], default="major") s = sub.add_parser( - "validate-json", help="Validate all JSON documents in a file or directory" + "validate-all", help="Validate all JSON documents in a file or directory" ) s.add_argument("--path", dest="scan_path", help="JSON file or directory to scan") @@ -152,16 +152,11 @@ def main(argv: list[str] | None = None) -> None: json.dump(out, sys.stdout, ensure_ascii=False, indent=2) sys.stdout.write("\n") return - elif args.op == "validate-json": + elif args.op == "validate-all": scan_path = args.scan_path or args.path if not scan_path: - parser.error("validate-json requires --path") + parser.error("validate-all requires --path") result = GtsJsonValidator(scan_path, ops.cfg).validate() - for issue in result.issues: - suffix = f"#{issue.index}" if issue.index is not None else "" - sys.stderr.write( - f"{issue.file}{suffix}: {issue.stage}: {issue.message}\n" - ) out = result.to_dict() elif args.op == "validate-id": out = ops.validate_id(args.gts_id).to_dict() diff --git a/gts/src/gts/_json_validation.py b/gts/src/gts/_json_validation.py index fa533d9..11e31f2 100644 --- a/gts/src/gts/_json_validation.py +++ b/gts/src/gts/_json_validation.py @@ -7,12 +7,12 @@ from pathlib import Path from typing import Any -from jsonschema.validators import validator_for - from .entities import GtsEntity, GtsFile -from .gts import GtsID +from .gts import GTS_PREFIX, GTS_URI_PREFIX, GtsID from .store import GtsStore +_X_GTS_REF_KEYWORD = "x-gts-ref" + @dataclass class GtsJsonValidationIssue: @@ -67,8 +67,12 @@ def __init__(self, path: str, cfg: Any) -> None: def validate(self) -> GtsJsonValidationResult: for file_path in self._json_files(): self._read_file(file_path) - self._validate_json_schemas() + self._check_schema_field_type() store = self._register_gts_entities() + schemas_count, instances_count = self._count_schema_instance() + self.result.schemas = schemas_count + self.result.instances = instances_count + self.result.gts_entities = schemas_count + instances_count self._validate_schemas(store) self._validate_instances(store) return self.result @@ -87,22 +91,35 @@ def _json_files(self) -> list[Path]: return [] files: list[Path] = [] - for root, dirs, names in os.walk(resolved, followlinks=True): - dirs[:] = [ - name for name in dirs if name not in {"node_modules", "dist", "build"} - ] - files.extend( - Path(root, name).resolve(strict=False) - for name in names - if Path(name).suffix.lower() == ".json" - ) - return sorted(set(files)) + seen: set[Path] = set() + for root, _dirs, names in os.walk(resolved, followlinks=True): + for name in names: + if Path(name).suffix.lower() == ".json": + rp = Path(root, name).resolve(strict=False) + if rp not in seen: + seen.add(rp) + files.append(rp) + return sorted(files) + + @staticmethod + def _is_gts_marker(text: str) -> bool: + return ( + GTS_PREFIX in text or GTS_URI_PREFIX in text or _X_GTS_REF_KEYWORD in text + ) def _read_file(self, file_path: Path) -> None: + try: + content_str = file_path.read_text(encoding="utf-8") + except Exception as error: # noqa: BLE001 - report this document and continue + self._issue(file_path, "json", str(error)) + return + + if not self._is_gts_marker(content_str): + return self.result.files += 1 + try: - with file_path.open(encoding="utf-8") as source: - content = json.load(source) + content = json.loads(content_str) except Exception as error: # noqa: BLE001 - report this document and continue self._issue(file_path, "json", str(error)) return @@ -120,15 +137,14 @@ def _read_file(self, file_path: Path) -> None: ) ) - def _validate_json_schemas(self) -> None: + def _check_schema_field_type(self) -> None: for entity in self.entities: content = entity.content - if not isinstance(content, dict) or "$schema" not in content: + if not isinstance(content, dict): continue - try: - validator_for(content).check_schema(content) - except Exception as error: # noqa: BLE001 - report this document and continue - self._issue(entity, "json-schema", str(error)) + schema_val = content.get("$schema") + if schema_val is not None and not isinstance(schema_val, str): + self._issue(entity, "json-schema", "$schema must be a string") def _register_gts_entities(self) -> GtsStore: store = GtsStore(reader=None) # type: ignore[arg-type] @@ -138,9 +154,12 @@ def _register_gts_entities(self) -> GtsStore: continue key = self._registry_key(entity) if key is None: - self._issue( - entity, "registry", "GTS-related document has no registrable GTS ID" - ) + if entity.is_schema: + self._issue( + entity, + "registry", + "GTS schema has a malformed or non-GTS $id", + ) continue if not entity.is_schema and entity.selected_entity_field is None: raw_id = entity.raw_id @@ -152,17 +171,28 @@ def _register_gts_entities(self) -> GtsStore: continue keys.add(key) store.register(entity) - self.result.gts_entities += 1 + return store + + def _count_schema_instance(self) -> tuple[int, int]: + schemas = 0 + instances = 0 + for entity in self.entities: + if self._registry_key(entity) is None: + continue if entity.is_schema: - self.result.schemas += 1 + schemas += 1 else: - self.result.instances += 1 - return store + instances += 1 + return schemas, instances @staticmethod def _is_gts_related(value: Any) -> bool: if isinstance(value, str): - return "gts." in value + return ( + GTS_PREFIX in value + or GTS_URI_PREFIX in value + or _X_GTS_REF_KEYWORD in value + ) if isinstance(value, dict): return any( GtsJsonValidator._is_gts_related(item) for item in value.values() @@ -187,43 +217,58 @@ def _registry_key(entity: GtsEntity) -> str | None: return None def _validate_schemas(self, store: GtsStore) -> None: - schemas = sorted( - ( - entity - for entity in self.entities - if entity.is_schema - and entity.gts_id - and store.get(entity.gts_id.id) is entity - ), - key=self._schema_depth, - ) - for stage, depth in (("base-type", 1), ("derived-type", None)): - for entity in schemas: - if (depth == 1) != (self._schema_depth(entity) == 1): - continue - gts_id = entity.gts_id - if not gts_id: - continue - try: - store.validate_schema(gts_id.id) - except Exception as error: # noqa: BLE001 - report this document and continue - self._issue(entity, stage, str(error)) + pending: list[tuple[int, str, str, int | None, GtsEntity]] = [] + for entity in self.entities: + if not entity.is_schema or not entity.gts_id: + continue + gid = entity.gts_id + if store.get(gid.id) is not entity: + continue + depth = len(gid.gts_id_segments) + file = entity.file.path if entity.file else entity.label + pending.append((depth, gid.id, file, entity.list_sequence, entity)) + pending.sort(key=lambda t: (t[0], t[1], t[2], t[3] if t[3] is not None else -1)) + + for depth, _gts_id, _file, _idx, entity in pending: + stage = "base-type" if depth <= 1 else "derived-type" + try: + store.validate_schema(entity.gts_id.id) # type: ignore[union-attr] + except Exception as error: # noqa: BLE001 - report this document and continue + self._issue(entity, stage, str(error)) @staticmethod def _schema_depth(entity: GtsEntity) -> int: return len(entity.gts_id.gts_id_segments) if entity.gts_id else 0 + @staticmethod + def _entity_depth(entity: GtsEntity) -> int: + if entity.gts_id: + return len(entity.gts_id.gts_id_segments) + if entity.type_id and GtsID.is_valid(entity.type_id): + return len(GtsID(entity.type_id).gts_id_segments) + return 0 + def _validate_instances(self, store: GtsStore) -> None: + pending: list[tuple[int, str, str, int | None, str]] = [] for entity in self.entities: + if entity.is_schema: + continue key = self._registry_key(entity) - if ( - entity.is_schema - or key is None - or not self._is_gts_related(entity.content) - ): + if key is None: + continue + depth = self._entity_depth(entity) + gts_id_str = entity.gts_id.id if entity.gts_id else "" + file = entity.file.path if entity.file else entity.label + pending.append((depth, gts_id_str, file, entity.list_sequence, key)) + pending.sort(key=lambda t: (t[0], t[1], t[2], t[3] if t[3] is not None else -1)) + + for _depth, _gts_id, _file, _idx, registry_key in pending: + # Find the entity for error reporting + entity = store.get(registry_key) + if entity is None: continue try: - store.validate_instance(key) + store.validate_instance(registry_key) except Exception as error: # noqa: BLE001 - report this document and continue self._issue(entity, "instance", str(error)) diff --git a/tests/test_json_validation.py b/tests/test_json_validation.py index 7f06f81..c8e9d2b 100644 --- a/tests/test_json_validation.py +++ b/tests/test_json_validation.py @@ -6,7 +6,7 @@ def test_validate_json_reports_all_document_errors(tmp_path): - (tmp_path / "broken.json").write_text("{", encoding="utf-8") + (tmp_path / "broken.json").write_text('{ "id": "gts.broken', encoding="utf-8") (tmp_path / "invalid-schema.json").write_text( json.dumps( { @@ -36,7 +36,6 @@ def test_validate_json_reports_all_document_errors(tmp_path): assert not result.ok assert {issue.stage for issue in result.issues} >= { "json", - "json-schema", "instance", } @@ -66,14 +65,173 @@ def test_validate_json_registers_type_only_instances(tmp_path): assert result.instances == 1 -def test_validate_json_cli_emits_issues_to_stderr(tmp_path, capsys): +def test_validate_json_cli_outputs_json_only(tmp_path, capsys): input_path = tmp_path / "broken.json" - input_path.write_text("{", encoding="utf-8") + input_path.write_text('{ "id": "gts.broken', encoding="utf-8") - main(["validate-json", "--path", str(input_path)]) + main(["validate-all", "--path", str(input_path)]) captured = capsys.readouterr() output = json.loads(captured.out) assert output["ok"] is False assert output["issues"][0]["file"] == str(input_path) - assert f"{input_path}: json:" in captured.err + assert captured.err == "" + + +def test_malformed_schema_id_is_reported(tmp_path): + malformed = json.dumps({ + "$id": "gts://gtx.cli.core.test.bad.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + }) + (tmp_path / "bad.schema.json").write_text(malformed, encoding="utf-8") + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert not result.ok, "malformed GTS id should fail" + assert result.gts_entities == 0 + assert any( + i.stage == "registry" and "malformed" in i.message for i in result.issues + ), f"expected a malformed-id diagnostic, got: {result.issues}" + + +def test_incidental_prefix_mention_is_not_registered(tmp_path): + doc = json.dumps({"description": "see gts.foo.bar for details", "value": 42}) + (tmp_path / "unrelated.json").write_text(doc, encoding="utf-8") + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert result.ok, f"incidental mention should not fail: {result.issues}" + assert result.documents == 1 + assert result.gts_entities == 0 + assert not result.issues + + +def test_duplicate_entity_is_reported(tmp_path): + schema = json.dumps({ + "$id": "gts://gts.cli.core.test.base.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }) + (tmp_path / "a.schema.json").write_text(schema, encoding="utf-8") + (tmp_path / "b.schema.json").write_text(schema, encoding="utf-8") + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert not result.ok, "duplicate ids should fail" + assert any( + "Duplicate" in i.message for i in result.issues + ), f"expected a duplicate diagnostic, got: {result.issues}" + + +def test_non_gts_files_are_ignored(tmp_path): + schema = json.dumps({ + "$id": "gts://gts.cli.core.test.base.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }) + (tmp_path / "base.schema.json").write_text(schema, encoding="utf-8") + (tmp_path / "package.json").write_text( + '{"name": "pkg", "version": "1.0.0"}', encoding="utf-8" + ) + nm = tmp_path / "node_modules" + nm.mkdir() + (nm / "broken.json").write_text("{ this is not json ", encoding="utf-8") + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert result.ok, f"non-GTS files must be ignored: {result.issues}" + assert result.files == 1, "only the GTS schema should be processed" + assert result.schemas == 1 + + +def test_marker_heuristic_matches_expected_combinations(): + assert GtsJsonValidator._is_gts_marker('{"id": "gts.x.y.z.t.v1~a.b.c.d.v1.0"}') + assert GtsJsonValidator._is_gts_marker('{"$id": "gts://gts.x.y.z.t.v1~"}') + assert GtsJsonValidator._is_gts_marker( + '{"properties": {"p": {"x-gts-ref": "..."}}}' + ) + assert not GtsJsonValidator._is_gts_marker( + '{"name": "widgets", "version": "1.0.0"}' + ) + + +def test_schema_errors_ordered_by_depth_then_gts_id(tmp_path): + base_schema = json.dumps({ + "$id": "gts://gts.cli.core.test.base.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }) + (tmp_path / "base.schema.json").write_text(base_schema, encoding="utf-8") + + def invalid_base(seg): + return json.dumps({ + "$id": f"gts://gts.cli.core.test.{seg}.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "invalid_type", + }) + + invalid_leaf = json.dumps({ + "$id": "gts://gts.cli.core.test.base.v1~cli.core.test.leaf.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "invalid_type", + }) + + (tmp_path / "0_mmm.schema.json").write_text(invalid_base("mmm"), encoding="utf-8") + (tmp_path / "a_leaf.schema.json").write_text(invalid_leaf, encoding="utf-8") + (tmp_path / "z_aaa.schema.json").write_text(invalid_base("aaa"), encoding="utf-8") + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + schema_issues = [ + i for i in result.issues if i.stage in ("base-type", "derived-type") + ] + + assert len(schema_issues) == 3, f"issues: {result.issues}" + assert schema_issues[0].stage == "base-type" + assert schema_issues[0].file.endswith("z_aaa.schema.json") + assert schema_issues[1].stage == "base-type" + assert schema_issues[1].file.endswith("0_mmm.schema.json") + assert schema_issues[2].stage == "derived-type" + assert schema_issues[2].file.endswith("a_leaf.schema.json") + + +def test_instance_errors_ordered_by_gts_id(tmp_path): + base_schema = json.dumps({ + "$id": "gts://gts.cli.core.test.base.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }) + (tmp_path / "base.schema.json").write_text(base_schema, encoding="utf-8") + + def invalid_instance(seg): + return json.dumps({ + "id": f"gts.cli.core.test.base.v1~cli.app._.{seg}.v1.0" + }) + + (tmp_path / "z_alpha.json").write_text( + invalid_instance("alpha"), encoding="utf-8" + ) + (tmp_path / "a_zeta.json").write_text( + invalid_instance("zeta"), encoding="utf-8" + ) + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + instance_issues = [i for i in result.issues if i.stage == "instance"] + + assert len(instance_issues) == 2, f"issues: {result.issues}" + assert instance_issues[0].file.endswith("z_alpha.json"), ( + f"alpha should be first: {instance_issues}" + ) + assert instance_issues[1].file.endswith("a_zeta.json"), ( + f"zeta should be second: {instance_issues}" + ) From c2a108de6b21912bf396ac871cab7083f8f1412a Mon Sep 17 00:00:00 2001 From: Artfizer Date: Thu, 10 Sep 2026 02:56:27 +0300 Subject: [PATCH 05/13] feat(cli): add global --exclude option for directory scanning Add a global `--exclude` option (alongside `--path`) that accepts a comma-separated list of directory names to skip during recursive file scanning. Defaults to `node_modules,dist,build,.git,target`. The parsed list is threaded through GtsOps (and reload_from_path) into GtsFileReader, and into GtsJsonValidator for validate-all. The module constant is renamed EXCLUDE_LIST -> DEFAULT_EXCLUDE_LIST and used as the per-instance fallback via a new `exclude` parameter on the reader and validator. Signed-off-by: Artfizer --- gts/src/gts/_cli.py | 25 +++++++++- gts/src/gts/_json_validation.py | 83 ++++++++++++++++++++++----------- gts/src/gts/files_reader.py | 20 ++++++-- gts/src/gts/ops.py | 10 +++- tests/test_json_validation.py | 6 ++- 5 files changed, 108 insertions(+), 36 deletions(-) diff --git a/gts/src/gts/_cli.py b/gts/src/gts/_cli.py index 398a3c4..db7dd34 100644 --- a/gts/src/gts/_cli.py +++ b/gts/src/gts/_cli.py @@ -19,6 +19,14 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument( "--path", help="Path to json and schema files or directories (global default)" ) + p.add_argument( + "--exclude", + default="node_modules,dist,build,.git,target", + help=( + "Comma-separated directory names to exclude when scanning " + "(default: node_modules,dist,build,.git,target)" + ), + ) sub = p.add_subparsers(dest="op", required=True) s = sub.add_parser("validate-id", help="Validate a GTS ID format") @@ -122,8 +130,16 @@ def main(argv: list[str] | None = None) -> None: ) try: + # Parse the comma-separated --exclude option into a list of dir names + exclude = [e.strip() for e in (args.exclude or "").split(",") if e.strip()] + # Helper to create GtsOps with common arguments - ops = GtsOps(path=args.path, config=args.config, verbose=args.verbose) + ops = GtsOps( + path=args.path, + config=args.config, + verbose=args.verbose, + exclude=exclude, + ) if args.op == "server": server = GtsHttpServer(ops=ops) @@ -156,8 +172,13 @@ def main(argv: list[str] | None = None) -> None: scan_path = args.scan_path or args.path if not scan_path: parser.error("validate-all requires --path") - result = GtsJsonValidator(scan_path, ops.cfg).validate() + result = GtsJsonValidator(scan_path, ops.cfg, exclude=exclude).validate() out = result.to_dict() + json.dump(out, sys.stdout, ensure_ascii=False, indent=2) + sys.stdout.write("\n") + if not result.ok: + raise SystemExit(1) + return elif args.op == "validate-id": out = ops.validate_id(args.gts_id).to_dict() elif args.op == "parse-id": diff --git a/gts/src/gts/_json_validation.py b/gts/src/gts/_json_validation.py index 11e31f2..3f070ad 100644 --- a/gts/src/gts/_json_validation.py +++ b/gts/src/gts/_json_validation.py @@ -8,6 +8,7 @@ from typing import Any from .entities import GtsEntity, GtsFile +from .files_reader import DEFAULT_EXCLUDE_LIST from .gts import GTS_PREFIX, GTS_URI_PREFIX, GtsID from .store import GtsStore @@ -58,9 +59,10 @@ def to_dict(self) -> dict[str, Any]: class GtsJsonValidator: - def __init__(self, path: str, cfg: Any) -> None: + def __init__(self, path: str, cfg: Any, exclude: list[str] | None = None) -> None: self.path = Path(path).expanduser() self.cfg = cfg + self.exclude = list(exclude) if exclude else list(DEFAULT_EXCLUDE_LIST) self.result = GtsJsonValidationResult() self.entities: list[GtsEntity] = [] @@ -91,14 +93,38 @@ def _json_files(self) -> list[Path]: return [] files: list[Path] = [] - seen: set[Path] = set() - for root, _dirs, names in os.walk(resolved, followlinks=True): + seen_files: set[Path] = set() + seen_dirs: set[tuple[int, int]] = set() + walk_errors: list[OSError] = [] + + def _on_walk_error(err: OSError) -> None: + walk_errors.append(err) + + for root, dirs, names in os.walk( + resolved, followlinks=True, onerror=_on_walk_error + ): + # Prevent symlink cycles by tracking visited directory identities + root_stat = os.stat(root) + dir_id = (root_stat.st_dev, root_stat.st_ino) + if dir_id in seen_dirs: + dirs.clear() + continue + seen_dirs.add(dir_id) + + # Prune excluded directories (defaults to DEFAULT_EXCLUDE_LIST, + # overridable via the CLI --exclude option) + dirs[:] = [d for d in dirs if d not in self.exclude] + for name in names: if Path(name).suffix.lower() == ".json": rp = Path(root, name).resolve(strict=False) - if rp not in seen: - seen.add(rp) + if rp not in seen_files: + seen_files.add(rp) files.append(rp) + + for err in walk_errors: + self._issue(resolved, "discovery", f"Traversal error: {err}") + return sorted(files) @staticmethod @@ -185,22 +211,28 @@ def _count_schema_instance(self) -> tuple[int, int]: instances += 1 return schemas, instances - @staticmethod - def _is_gts_related(value: Any) -> bool: - if isinstance(value, str): - return ( - GTS_PREFIX in value - or GTS_URI_PREFIX in value - or _X_GTS_REF_KEYWORD in value - ) - if isinstance(value, dict): - return any( - GtsJsonValidator._is_gts_related(item) for item in value.values() - ) - if isinstance(value, list): - return any(GtsJsonValidator._is_gts_related(item) for item in value) + def _is_gts_related(self, value: Any) -> bool: + if not isinstance(value, dict): + return False + # Check configured identifier fields for GTS IDs. + # Accept valid IDs and also detect likely-but-malformed ones + # (gts:// or gts. prefix) so they get diagnosed during registration + # rather than silently skipped. + for f in self.cfg.entity_id_fields: + v = value.get(f) + if isinstance(v, str) and self._looks_gts(v): + return True + for f in self.cfg.schema_id_fields: + v = value.get(f) + if isinstance(v, str) and self._looks_gts(v): + return True return False + @staticmethod + def _looks_gts(v: str) -> bool: + normalized = v.removeprefix(GTS_URI_PREFIX) + return normalized.startswith(GTS_PREFIX) or v.startswith(GTS_URI_PREFIX) + @staticmethod def _registry_key(entity: GtsEntity) -> str | None: if entity.is_schema and entity.gts_id: @@ -249,24 +281,23 @@ def _entity_depth(entity: GtsEntity) -> int: return 0 def _validate_instances(self, store: GtsStore) -> None: - pending: list[tuple[int, str, str, int | None, str]] = [] + pending: list[tuple[int, str, str, int | None, str, GtsEntity]] = [] for entity in self.entities: if entity.is_schema: continue key = self._registry_key(entity) if key is None: continue + # Skip rejected duplicates: only validate the registered entity + if store.get(key) is not entity: + continue depth = self._entity_depth(entity) gts_id_str = entity.gts_id.id if entity.gts_id else "" file = entity.file.path if entity.file else entity.label - pending.append((depth, gts_id_str, file, entity.list_sequence, key)) + pending.append((depth, gts_id_str, file, entity.list_sequence, key, entity)) pending.sort(key=lambda t: (t[0], t[1], t[2], t[3] if t[3] is not None else -1)) - for _depth, _gts_id, _file, _idx, registry_key in pending: - # Find the entity for error reporting - entity = store.get(registry_key) - if entity is None: - continue + for _depth, _gts_id, _file, _idx, registry_key, entity in pending: try: store.validate_instance(registry_key) except Exception as error: # noqa: BLE001 - report this document and continue diff --git a/gts/src/gts/files_reader.py b/gts/src/gts/files_reader.py index 66cd24d..f309f95 100644 --- a/gts/src/gts/files_reader.py +++ b/gts/src/gts/files_reader.py @@ -12,7 +12,9 @@ from .entities import DEFAULT_GTS_CONFIG, GtsConfig, GtsEntity, GtsFile from .store import GtsReader -EXCLUDE_LIST = ["node_modules", "dist", "build"] +# Default directory names skipped during recursive scanning. The CLI --exclude +# option overrides this per invocation. +DEFAULT_EXCLUDE_LIST = ["node_modules", "dist", "build", ".git", "target"] logger = logging.getLogger(__name__) @@ -20,13 +22,20 @@ class GtsFileReader(GtsReader): """Reads GTS entities from JSON and YAML files in directories specified by path.""" - def __init__(self, path: str | list[str], cfg: GtsConfig | None = None) -> None: + def __init__( + self, + path: str | list[str], + cfg: GtsConfig | None = None, + exclude: list[str] | None = None, + ) -> None: """ Initialize FileReader with one or more paths. Args: path: Single path string or list of paths (files or directories) cfg: GtsConfig for entity ID extraction (defaults to DEFAULT_GTS_CONFIG) + exclude: Directory names to skip while scanning (defaults to + DEFAULT_EXCLUDE_LIST) """ self.paths: list[Path] = [] if isinstance(path, str): @@ -35,6 +44,7 @@ def __init__(self, path: str | list[str], cfg: GtsConfig | None = None) -> None: self.paths = [Path(os.path.expanduser(p)) for p in path] self.cfg = cfg or DEFAULT_GTS_CONFIG + self.exclude = list(exclude) if exclude else list(DEFAULT_EXCLUDE_LIST) self._files: list[Path] = [] self._current_index = 0 self._current_file_entities: list[GtsEntity] = [] @@ -61,9 +71,9 @@ def _collect_files(self) -> None: elif resolved_path.is_dir(): # Recursively scan for all valid file types, following symlinks for root, dirs, files in os.walk(resolved_path, followlinks=True): - for exclude in EXCLUDE_LIST: - if exclude in dirs: - dirs.remove(exclude) + for excluded in self.exclude: + if excluded in dirs: + dirs.remove(excluded) for fname in files: ext = os.path.splitext(fname)[1].lower() if ext in valid_extensions: diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index 4954c83..99e7856 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -297,11 +297,17 @@ def __init__( path: str | builtins.list[str] | None = None, config: str | None = None, verbose: int = 0, + exclude: builtins.list[str] | None = None, ) -> None: self.verbose = verbose self.cfg = self._load_config(config) self.path: str | list[str] | None = path - self._reader = GtsFileReader(self.path, cfg=self.cfg) if self.path else None + self.exclude = exclude + self._reader = ( + GtsFileReader(self.path, cfg=self.cfg, exclude=self.exclude) + if self.path + else None + ) self.store = GtsStore(self._reader) if self._reader else GtsStore(reader=None) # type: ignore[arg-type] @staticmethod @@ -345,7 +351,7 @@ def _load_config(self, config_path: str | None) -> GtsConfig: def reload_from_path(self, path: str | builtins.list[str]) -> None: self.path = path - self._reader = GtsFileReader(self.path, cfg=self.cfg) + self._reader = GtsFileReader(self.path, cfg=self.cfg, exclude=self.exclude) self.store = GtsStore(self._reader) def add_entity( diff --git a/tests/test_json_validation.py b/tests/test_json_validation.py index c8e9d2b..8ada971 100644 --- a/tests/test_json_validation.py +++ b/tests/test_json_validation.py @@ -1,5 +1,7 @@ import json +import pytest + from gts._cli import main from gts.entities import DEFAULT_GTS_CONFIG from gts._json_validation import GtsJsonValidator @@ -69,7 +71,9 @@ def test_validate_json_cli_outputs_json_only(tmp_path, capsys): input_path = tmp_path / "broken.json" input_path.write_text('{ "id": "gts.broken', encoding="utf-8") - main(["validate-all", "--path", str(input_path)]) + with pytest.raises(SystemExit) as exc_info: + main(["validate-all", "--path", str(input_path)]) + assert exc_info.value.code == 1 captured = capsys.readouterr() output = json.loads(captured.out) From 7c86a1195be14857f7bcdaa7a5b456992d31004c Mon Sep 17 00:00:00 2001 From: Artfizer Date: Thu, 10 Sep 2026 23:39:12 +0300 Subject: [PATCH 06/13] fix(files_reader): prevent symlink traversal cycles Track visited directory identities while following links and stop recursing into directories already encountered. Prune excluded directories in-place so os.walk does not descend into them. Signed-off-by: Artfizer --- gts/src/gts/files_reader.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/gts/src/gts/files_reader.py b/gts/src/gts/files_reader.py index f309f95..f8ca43e 100644 --- a/gts/src/gts/files_reader.py +++ b/gts/src/gts/files_reader.py @@ -55,6 +55,7 @@ def _collect_files(self) -> None: """Collect all JSON and YAML files from the specified paths, following symlinks.""" valid_extensions = {".json", ".jsonc", ".gts", ".yaml", ".yml"} seen: set[str] = set() + seen_dirs: set[tuple[int, int]] = set() collected: list[Path] = [] for path in self.paths: @@ -71,9 +72,15 @@ def _collect_files(self) -> None: elif resolved_path.is_dir(): # Recursively scan for all valid file types, following symlinks for root, dirs, files in os.walk(resolved_path, followlinks=True): - for excluded in self.exclude: - if excluded in dirs: - dirs.remove(excluded) + # Prevent symlink cycles by tracking visited directory identities + root_stat = os.stat(root) + dir_id = (root_stat.st_dev, root_stat.st_ino) + if dir_id in seen_dirs: + dirs.clear() + continue + seen_dirs.add(dir_id) + + dirs[:] = [d for d in dirs if d not in self.exclude] for fname in files: ext = os.path.splitext(fname)[1].lower() if ext in valid_extensions: From f570d178649678b621ef95f6cbf09febd5071d83 Mon Sep 17 00:00:00 2001 From: Artfizer Date: Thu, 10 Sep 2026 23:39:53 +0300 Subject: [PATCH 07/13] fix(schema): require gts URI form for schema IDs Reject schemas whose $id uses the bare gts. prefix at GtsEntity construction time, so registration and JSON validation enforce the same rule regardless of the validate flag. Update callers and tests to use the required gts:// schema URI form. Signed-off-by: Artfizer --- gts/src/gts/_json_validation.py | 2 ++ gts/src/gts/entities.py | 27 +++++++++++++++++++++++++-- gts/src/gts/ops.py | 24 +++++++----------------- tests/test_ops.py | 17 ++++++++++------- tests/test_server.py | 2 +- tests/test_store.py | 4 ++-- 6 files changed, 47 insertions(+), 29 deletions(-) diff --git a/gts/src/gts/_json_validation.py b/gts/src/gts/_json_validation.py index 3f070ad..46104b2 100644 --- a/gts/src/gts/_json_validation.py +++ b/gts/src/gts/_json_validation.py @@ -181,6 +181,8 @@ def _register_gts_entities(self) -> GtsStore: key = self._registry_key(entity) if key is None: if entity.is_schema: + # A schema with a plain gts. $id (not gts://) yields no + # gts_id at the core entity layer, so it lands here. self._issue( entity, "registry", diff --git a/gts/src/gts/entities.py b/gts/src/gts/entities.py index 4c92028..171e68e 100644 --- a/gts/src/gts/entities.py +++ b/gts/src/gts/entities.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any -from .gts import GtsID +from .gts import GTS_PREFIX, GTS_URI_PREFIX, GtsID from .schema_cast import GtsEntityCastResult, SchemaCastError if TYPE_CHECKING: @@ -130,7 +130,15 @@ def __init__( self.type_id and GtsID.is_valid(self.type_id) ): idv = self.type_id - self.gts_id = GtsID(idv) if idv and GtsID.is_valid(idv) else None + # Enforce gts:// URI form for schema $id at the core layer. Per + # gts-spec, a schema must place its identifier in $id as a gts:// + # URI, not the bare gts. prefix. Leaving gts_id as None makes every + # client (server, batch validator, direct callers) reject it + # uniformly, mirroring gts-go extract.go and gts-rust entities.rs. + if self.is_schema and self._schema_id_uses_plain_prefix(): + self.gts_id = None + else: + self.gts_id = GtsID(idv) if idv and GtsID.is_valid(idv) else None # Set label if self.file and self.list_sequence is not None: @@ -287,6 +295,21 @@ def _get_field_value(self, field: str) -> str | None: return v return None + def _schema_id_uses_plain_prefix(self) -> bool: + """Return True if the raw schema $id uses the bare gts. prefix. + + Schemas must express their identifier as a gts:// URI in $id. A raw + $id that starts with the plain gts. prefix (without gts://) is invalid + and must not yield a gts_id. + """ + if not isinstance(self.content, dict): + return False + raw = self.content.get("$id") + if not isinstance(raw, str): + return False + raw = raw.strip() + return raw.startswith(GTS_PREFIX) and not raw.startswith(GTS_URI_PREFIX) + def _first_non_empty_field(self, fields: list[str]) -> tuple[str, str] | None: """Find first non-empty field value in order. diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index 99e7856..a86376d 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -370,27 +370,17 @@ def add_entity( is_type_schema=False, ) - # Schemas MUST have a valid GTS ID + # Schemas MUST have a valid GTS ID. The core entity layer already + # rejects a plain gts. $id (without gts://) by leaving gts_id as None, + # so this single guard covers both the malformed and the wrong-prefix + # cases uniformly for every client. if entity.is_schema and not entity.gts_id: return GtsAddEntityResult( - ok=False, error="Unable to detect GTS ID in schema" + ok=False, + error="Unable to detect GTS ID in schema", + is_type_schema=entity.is_schema, ) - # Validate $id prefix for schemas: must use gts:// URI, not plain gts. - if entity.is_schema and validate: - raw_id = content.get("$id", "") - # Reject plain gts. prefix (without gts://) - if ( - isinstance(raw_id, str) - and raw_id.startswith("gts.") - and not raw_id.startswith("gts://") - ): - return GtsAddEntityResult( - ok=False, - error="Schema $id must use gts:// URI format, not plain gts. prefix", - is_type_schema=True, - ) - store_key = entity.gts_id.id if entity.is_schema else entity.raw_id previous = self.store.get(store_key) self.store.register(entity) diff --git a/tests/test_ops.py b/tests/test_ops.py index 95ca333..3e0fd90 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -7,7 +7,7 @@ SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "gts.x.test._.foo.v1~", + "$id": "gts://gts.x.test._.foo.v1~", "type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"], @@ -56,13 +56,16 @@ def test_add_schema_missing_gts_id(self, ops): assert result.ok is False assert "Unable to detect GTS ID" in result.error - def test_add_schema_plain_gts_prefix_rejected_when_validate(self, ops): + def test_add_schema_plain_gts_prefix_rejected(self, ops): + # A schema $id MUST use the gts:// URI form. A bare gts. prefix yields + # no gts_id at the core entity layer, so registration is rejected + # regardless of the validate flag. schema = dict(SCHEMA) schema["$id"] = "gts.x.test._.foo.v1~" - result = ops.add_entity(schema, validate=True) - # $id doesn't start with gts:// -> rejected only if raw $id startswith "gts." - assert result.ok is False - assert "gts:// URI format" in result.error + for validate in (False, True): + result = ops.add_entity(schema, validate=validate) + assert result.ok is False + assert "Unable to detect GTS ID in schema" in result.error def test_add_instance_without_id_field_rejected(self, ops): result = ops.add_entity({"name": "hi"}) @@ -88,7 +91,7 @@ def test_add_instance_validate_failure_restores_previous(self, ops): def test_add_schema_validate_basic_failure(self, ops): bad_schema = { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "gts.x.test._.foo.v1~", + "$id": "gts://gts.x.test._.foo.v1~", "type": "object", "x-gts-ref": "notgts.*", } diff --git a/tests/test_server.py b/tests/test_server.py index da5e1c0..9278f18 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -12,7 +12,7 @@ SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "gts.x.test._.foo.v1~", + "$id": "gts://gts.x.test._.foo.v1~", "type": "object", "properties": {"name": {"type": "string"}}, } diff --git a/tests/test_store.py b/tests/test_store.py index 3b03d29..778ccdc 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -245,7 +245,7 @@ def _create_store_with_schema_and_instance(self): schema = GtsEntity( content={ "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "gts.vendor.package.namespace.type.v1~", + "$id": "gts://gts.vendor.package.namespace.type.v1~", "type": "object", "properties": { "name": {"type": "string"}, @@ -316,7 +316,7 @@ def test_build_graph_simple(self): """Test building a simple graph.""" schema = GtsEntity( content={ - "$id": "gts.vendor.package.namespace.type.v1~", + "$id": "gts://gts.vendor.package.namespace.type.v1~", "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", }, From 38f33bf2e46c67853b8c84ffa96eb537516758e8 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Fri, 11 Sep 2026 11:21:53 +0300 Subject: [PATCH 08/13] fix: Fix Makefile virtualenv recovery Recreate stale or broken virtual environments before running quality checks, use a platform-aware Python executable path, and update the local install prerequisite accordingly. Signed-off-by: Artifizer --- Makefile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index f9f3f1b..1ac5666 100644 --- a/Makefile +++ b/Makefile @@ -11,10 +11,11 @@ SHELL := /bin/bash PYTHON_BOOTSTRAP ?= $(shell command -v python3 2>/dev/null || command -v python 2>/dev/null || echo python3) PY_ENV_DIR ?= .venv ifeq ($(OS),Windows_NT) -PYTHON ?= $(PY_ENV_DIR)/Scripts/python +PY_ENV_PYTHON := $(PY_ENV_DIR)/Scripts/python else -PYTHON ?= $(PY_ENV_DIR)/bin/python +PY_ENV_PYTHON := $(PY_ENV_DIR)/bin/python endif +PYTHON ?= $(PY_ENV_PYTHON) PY_ENV_STAMP := $(PY_ENV_DIR)/.stamp INSTALL_STAMP := $(PY_ENV_DIR)/.install-stamp LOCAL_DIST_DIR := dist-install-local @@ -39,10 +40,10 @@ help: # Create/update the virtual environment and install dev/test dependencies py-env: $(PY_ENV_STAMP) -$(PY_ENV_DIR)/bin/python: - $(PYTHON_BOOTSTRAP) -m venv $(PY_ENV_DIR) +$(PY_ENV_PYTHON): + $(PYTHON_BOOTSTRAP) -m venv --clear $(PY_ENV_DIR) -$(PY_ENV_STAMP): gts/pyproject.toml .gts-spec/tests/requirements.txt Makefile +$(PY_ENV_STAMP): $(PY_ENV_PYTHON) gts/pyproject.toml .gts-spec/tests/requirements.txt Makefile @echo "Creating/updating Python virtual environment in $(PY_ENV_DIR)..." $(PYTHON_BOOTSTRAP) -m venv $(PY_ENV_DIR) $(PYTHON) -m pip install --upgrade pip @@ -64,7 +65,7 @@ build: py-env $(PYTHON) -m build --outdir dist ./gts # Install the locally built wheel, equivalent to installing the published gts package -install-local: $(if $(filter $(PY_ENV_DIR)/bin/python,$(PYTHON)),$(PY_ENV_DIR)/bin/python) +install-local: $(if $(filter $(PY_ENV_PYTHON),$(PYTHON)),$(PY_ENV_PYTHON)) @rm -rf $(LOCAL_DIST_DIR) $(PYTHON) -m pip install --upgrade build $(PYTHON) -m build --outdir $(LOCAL_DIST_DIR) ./gts From 0cc86dae4be42a58696cdbbcc82aadbc92d695d7 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Fri, 11 Sep 2026 11:45:54 +0300 Subject: [PATCH 09/13] fix(server): close responses to avoid descriptor exhaustion Prevent idle keep-alive sockets from accumulating during spec-test runs. This keeps the server within macOS's default 256 file-descriptor limit and prevents subsequent requests from failing with status 0. Assert that the non-verbose request middleware closes response connections. Signed-off-by: Artifizer --- gts/src/gts/_server.py | 30 +++++++++++++++++++++++++++++- tests/test_server.py | 5 +++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/gts/src/gts/_server.py b/gts/src/gts/_server.py index 639ae16..89f4bf1 100644 --- a/gts/src/gts/_server.py +++ b/gts/src/gts/_server.py @@ -41,7 +41,9 @@ def __init__(self, app: FastAPI, verbose: int) -> None: async def dispatch(self, request, call_next): if not self.verbose: - return await call_next(request) + response = await call_next(request) + response.headers["connection"] = "close" + return response start = time.time() @@ -60,6 +62,7 @@ async def receive(): request = Request(request.scope, receive) response = await call_next(request) + response.headers["connection"] = "close" dur = (time.time() - start) * 1000.0 # Determine status color @@ -275,6 +278,18 @@ def _register_routes(self) -> None: methods=["POST"], summary="Validate instance by GTS ID", ) + app.add_api_route( + "/validate-json", + self.validate_json, + methods=["POST"], + summary="Validate unregistered JSON entity", + ) + app.add_api_route( + "/validate-json/{gts_type:path}", + self.validate_json_as_type, + methods=["POST"], + summary="Validate unregistered JSON instance against a type", + ) # Op #12 - validate type schema app.add_api_route( "/validate-type-schema", @@ -371,6 +386,19 @@ async def id_to_uuid(self, id: str = Query(..., alias="gts_id")) -> dict[str, An async def validate_instance(self, body: ValidateInstanceRequest) -> dict[str, Any]: return self.ops.validate_instance(body.instance_id).to_dict() + async def validate_json( + self, + body: dict[str, Any] = Body(...), # noqa: B008 - FastAPI dependency pattern + ) -> dict[str, Any]: + return self.ops.validate_json(body).to_dict() + + async def validate_json_as_type( + self, + gts_type: str, + body: dict[str, Any] = Body(...), # noqa: B008 - FastAPI dependency pattern + ) -> dict[str, Any]: + return self.ops.validate_json(body, explicit_type_id=gts_type).to_dict() + async def validate_type_schema( self, body: ValidateTypeSchemaRequest ) -> dict[str, Any]: diff --git a/tests/test_server.py b/tests/test_server.py index 9278f18..741a144 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -5,6 +5,7 @@ import asyncio import pytest +from fastapi.responses import JSONResponse from gts.ops import GtsOps from gts._server import GtsHttpServer, ValidateEntityRequest, _RequestLoggingMiddleware @@ -190,7 +191,7 @@ def test_dispatch_skips_when_not_verbose(self, server): middleware = _RequestLoggingMiddleware(server.app, verbose=0) async def call_next(request): - return "response-sentinel" + return JSONResponse({"ok": True}) result = run(middleware.dispatch(request=None, call_next=call_next)) - assert result == "response-sentinel" + assert result.headers["connection"] == "close" From da0c5df4bc811f0770ebf6adc3fa4b3603050a14 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Fri, 11 Sep 2026 12:19:09 +0300 Subject: [PATCH 10/13] feat(validation): validate transient JSON content Signed-off-by: Artifizer --- gts/src/gts/ops.py | 103 +++++++++++++++++- gts/src/gts/store.py | 244 ++++++++++++++++++++++++++----------------- tests/test_ops.py | 75 ++++++++++++- 3 files changed, 322 insertions(+), 100 deletions(-) diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index a86376d..8ae746c 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -152,6 +152,30 @@ def to_dict(self) -> dict[str, Any]: return result +@dataclass +class GtsJsonValidationResult: + """Result of validating an unregistered JSON entity.""" + + ok: bool + id: str = "" + type_id: str | None = None + is_type_schema: bool = False + error: str = "" + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "ok": self.ok, + "is_type_schema": self.is_type_schema, + } + if self.id: + result["id"] = self.id + if self.type_id: + result["type_id"] = self.type_id + if self.error: + result["error"] = self.error + return result + + @dataclass class GtsSchemaGraphResult: """Result of building a schema graph for an entity.""" @@ -366,7 +390,7 @@ def add_entity( ): return GtsAddEntityResult( ok=False, - error="Instance must have an id field", + error="Unable to detect GTS ID in instance entity: Instance must have an id field", is_type_schema=False, ) @@ -524,6 +548,83 @@ def uuid(self, gts_id: str) -> GtsUuidResult: g = GtsID(gts_id) return GtsUuidResult(id=g.id, uuid=str(g.to_uuid())) + def validate_json( + self, content: dict[str, Any], explicit_type_id: str | None = None + ) -> GtsJsonValidationResult: + """Validate JSON without retaining it in the registry.""" + entity = GtsEntity(content=content, cfg=self.cfg) + if explicit_type_id is not None: + try: + explicit_type = GtsID(explicit_type_id) + except ValueError: + if explicit_type_id.startswith(("gts.", "gts://")): + return GtsJsonValidationResult( + ok=False, + error=f"Explicit type '{explicit_type_id}' must be GTS Type schema", + ) + return GtsJsonValidationResult( + ok=False, error=f"Invalid GTS Type Schema ID: {explicit_type_id}" + ) + if not explicit_type.is_type: + return GtsJsonValidationResult( + ok=False, + error=f"Explicit type '{explicit_type_id}' must be GTS Type schema", + ) + explicit_type_id = explicit_type.id + if entity.is_schema: + return GtsJsonValidationResult( + ok=False, + is_type_schema=True, + error="Explicit type validation only accepts instance JSON", + ) + if entity.type_id and entity.type_id != explicit_type_id: + return GtsJsonValidationResult( + ok=False, + type_id=explicit_type_id, + error=( + f"Declared type '{entity.type_id}' does not match path type " + f"'{explicit_type_id}'" + ), + ) + entity.type_id = explicit_type_id + elif entity.is_schema and not entity.gts_id: + return GtsJsonValidationResult( + ok=False, + is_type_schema=True, + error="Unable to detect GTS ID in schema", + ) + elif not entity.is_schema and not entity.type_id: + return GtsJsonValidationResult( + ok=False, + error="Unable to determine instance type", + ) + + try: + if entity.is_schema: + self.store.validate_schema_content(entity.gts_id.id, content) # type: ignore[union-attr] + else: + self.store.validate_instance_content(content, entity.type_id) + except Exception as error: # noqa: BLE001 - converted to a result object at API boundary + error_message = str(error) + if entity.is_schema and "not found for chain validation" in error_message: + error_message = f"Parent GTS Type Schema not found: {error_message}" + elif explicit_type_id is not None and "not found in store" in error_message: + error_message = f"GTS Type Schema not found: {explicit_type_id}" + return GtsJsonValidationResult( + ok=False, + id=entity.gts_id.id if entity.gts_id else (entity.raw_id or ""), + type_id=entity.type_id, + is_type_schema=entity.is_schema, + error=error_message, + ) + + return GtsJsonValidationResult( + ok=True, + id=entity.gts_id.id if entity.gts_id else (entity.raw_id or ""), + type_id=entity.type_id, + is_type_schema=entity.is_schema, + ) + def validate_instance(self, gts_id: str) -> GtsValidationResult: try: self.store.validate_instance(gts_id) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 7ff55b7..72edaa3 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -302,11 +302,16 @@ def _validate_schema_x_gts_refs(self, gts_id: str) -> None: if not schema_entity.is_schema: raise ValueError(f"Entity '{gts_id}' is not a schema") + self._validate_schema_x_gts_refs_content(gts_id, schema_entity.content) + + def _validate_schema_x_gts_refs_content( + self, gts_id: str, schema_content: dict[str, Any] + ) -> None: logger.info(f"Validating schema x-gts-ref fields for {gts_id}") # Validate x-gts-ref constraints in the schema x_gts_ref_validator = XGtsRefValidator(store=self) - x_gts_ref_errors = x_gts_ref_validator.validate_schema(schema_entity.content) + x_gts_ref_errors = x_gts_ref_validator.validate_schema(schema_content) if x_gts_ref_errors: error_messages = [ f"{err.field_path}: {err.reason}" for err in x_gts_ref_errors @@ -317,7 +322,15 @@ def _validate_schema_x_gts_refs(self, gts_id: str) -> None: @staticmethod def _validate_gts_keywords(content: dict[str, Any]) -> None: - """Validate x-gts-final, x-gts-abstract, x-gts-traits, x-gts-traits-schema placement.""" + """Validate supported GTS extensions and their placement.""" + + top_level_keywords = { + "x-gts-final", + "x-gts-abstract", + "x-gts-traits", + "x-gts-traits-schema", + } + supported_keywords = top_level_keywords | {"x-gts-ref"} def _contains_key_recursive(value: Any, key: str) -> bool: if isinstance(value, dict): @@ -348,14 +361,20 @@ def _contains_key_recursive(value: Any, key: str) -> bool: "schema cannot declare both x-gts-final and x-gts-abstract as true" ) + def _validate_extensions(value: Any) -> None: + if isinstance(value, dict): + for key, nested_value in value.items(): + if key.startswith("x-gts-") and key not in supported_keywords: + raise ValueError(f"Unsupported GTS extension keyword: {key}") + _validate_extensions(nested_value) + elif isinstance(value, list): + for item in value: + _validate_extensions(item) + + _validate_extensions(content) + # Check that x-gts-final/x-gts-abstract/x-gts-traits/x-gts-traits-schema # appear only at the top level - top_level_keywords = { - "x-gts-final", - "x-gts-abstract", - "x-gts-traits", - "x-gts-traits-schema", - } for key, value in content.items(): if key in top_level_keywords: continue @@ -371,7 +390,9 @@ def _content_is_abstract(content: dict[str, Any]) -> bool: def _content_is_final(content: dict[str, Any]) -> bool: return content.get("x-gts-final") is True - def _validate_schema_chain(self, gts_id: str) -> None: + def _validate_schema_chain( + self, gts_id: str, transient_schema: dict[str, Any] | None = None + ) -> None: """Validate OP#12: schema derivation chain compatibility.""" gid = GtsID(gts_id) segments = gid.gts_id_segments @@ -394,6 +415,13 @@ def _validate_schema_chain(self, gts_id: str) -> None: base_entity = self.get(base_id) derived_entity = self.get(derived_id) + derived_content = ( + transient_schema + if transient_schema is not None and derived_id == gts_id + else derived_entity.content + if derived_entity + else None + ) # Check x-gts-final: if the base type is final, derivation is not allowed. if ( @@ -413,14 +441,14 @@ def _validate_schema_chain(self, gts_id: str) -> None: raise ValueError( f"Base schema '{base_id}' not found for chain validation" ) - if not derived_entity or not isinstance(derived_entity.content, dict): - raise ValueError( + if not isinstance(derived_content, dict): + raise TypeError( f"Derived schema '{derived_id}' not found for chain validation" ) # Resolve both schemas (inline $refs) base_resolved = self._resolve_schema_refs(base_entity.content) - derived_resolved = self._resolve_schema_refs(derived_entity.content) + derived_resolved = self._resolve_schema_refs(derived_content) # Validate derivation compatibility (OP#12): accepted-instance-set # inclusion on declared schemas plus GTS admission rules. @@ -504,7 +532,9 @@ def _inline_refs( ] return node - def _build_effective_traits(self, gts_id: str) -> traits.EffectiveTraits: + def _build_effective_traits( + self, gts_id: str, transient_schema: dict[str, Any] | None = None + ) -> traits.EffectiveTraits: """Build OP#13 EffectiveTraits by walking the type's chain (root -> leaf).""" gid = GtsID(gts_id) segments = gid.gts_id_segments @@ -520,9 +550,15 @@ def _build_effective_traits(self, gts_id: str) -> traits.EffectiveTraits: for schema_id in chain_ids: entity = self.get(schema_id) - if not entity or not isinstance(entity.content, dict): + content = ( + transient_schema + if transient_schema is not None and schema_id == gts_id + else entity.content + if entity + else None + ) + if not isinstance(content, dict): continue - content = entity.content level_schemas: list[Any] = [] traits.collect_trait_schema_from_value(content, level_schemas) @@ -537,18 +573,32 @@ def _build_effective_traits(self, gts_id: str) -> traits.EffectiveTraits: traits.merge_rfc7396_into(merged_traits, level_traits) leaf = self.get(chain_ids[-1]) if chain_ids else None + leaf_content = ( + transient_schema + if transient_schema is not None and chain_ids[-1] == gts_id + else leaf.content + if leaf + else None + ) dialect = None - if leaf and isinstance(leaf.content, dict): - ds = leaf.content.get("$schema") + if isinstance(leaf_content, dict): + ds = leaf_content.get("$schema") if isinstance(ds, str): dialect = ds return traits.build_effective_traits(trait_schemas, merged_traits, dialect) - def _validate_traits(self, gts_id: str, is_abstract: bool) -> None: + def _validate_traits( + self, + gts_id: str, + is_abstract: bool, + transient_schema: dict[str, Any] | None = None, + ) -> None: """Validate OP#13: schema traits for a type.""" - effective = self._build_effective_traits(gts_id) - errors = effective.validate(check_unresolved=not is_abstract) + effective = self._build_effective_traits(gts_id, transient_schema) + errors = effective.validate( + check_unresolved=not is_abstract, reference_store=self + ) if errors: raise ValueError( f"Schema '{gts_id}' trait validation failed: " + "; ".join(errors) @@ -598,34 +648,14 @@ def validate_schema_basic(self, gts_id: str) -> None: # 3. Validate GTS keywords (x-gts-final, x-gts-abstract, placement) self._validate_gts_keywords(schema_content) - def validate_schema(self, gts_id: str) -> None: - """ - Full schema validation including: - 1. JSON Schema meta-schema validation - 2. x-gts-ref field validation - 3. GTS keyword validation (x-gts-final, x-gts-abstract, placement) - 4. Schema chain derivation validation (OP#12) - - Args: - gts_id: The GTS ID of the schema to validate - """ - if not gts_id.endswith("~"): + def validate_schema_content( + self, gts_id: str, schema_content: dict[str, Any] + ) -> None: + """Validate a schema using the registry only for its dependencies.""" + schema_id = GtsID(gts_id) + if not schema_id.is_type: raise ValueError(f"ID '{gts_id}' is not a schema (must end with '~')") - schema_entity = self.get(gts_id) - if not schema_entity: - raise StoreGtsSchemaNotFound(gts_id) - - if not schema_entity.is_schema: - raise ValueError(f"Entity '{gts_id}' is not a schema") - - schema_content = schema_entity.content - if not isinstance(schema_content, dict): - raise ValueError( # noqa: TRY004 - keep ValueError for API compatibility - f"Schema '{gts_id}' content must be a dictionary" - ) - - # Issue #25: strict check, no GTS IDs in $schema meta_schema_url = schema_content.get("$schema") if ( meta_schema_url @@ -636,21 +666,12 @@ def validate_schema(self, gts_id: str) -> None: f"Invalid $schema URL '{meta_schema_url}': must be a standard JSON Schema URL, not a GTS ID" ) - logger.info(f"Validating schema {gts_id}") - - # 1. Validate $ref fields - must be local (#...) or gts:// URIs + logger.info(f"Validating schema {schema_id.id}") self._validate_schema_refs(schema_content, "") - - # 2. Validate x-gts-ref fields - self._validate_schema_x_gts_refs(gts_id) - - # 3. Validate GTS keywords (x-gts-final, x-gts-abstract, placement) + self._validate_schema_x_gts_refs_content(schema_id.id, schema_content) self._validate_gts_keywords(schema_content) + self._validate_schema_chain(schema_id.id, schema_content) - # 4. Validate schema derivation chain (OP#12) - self._validate_schema_chain(gts_id) - - # 5. Validate against JSON Schema meta-schema try: from jsonschema import Draft7Validator from jsonschema.validators import validator_for @@ -661,15 +682,75 @@ def validate_schema(self, gts_id: str) -> None: else: Draft7Validator.check_schema(schema_content) - logger.info(f"Schema {gts_id} passed JSON Schema meta-schema validation") - except Exception as e: + logger.info( + f"Schema {schema_id.id} passed JSON Schema meta-schema validation" + ) + except Exception as error: raise ValueError( - f"JSON Schema validation failed for '{gts_id}': {e!s}" - ) from e + f"JSON Schema validation failed for '{schema_id.id}': {error!s}" + ) from error + + self._validate_traits( + schema_id.id, + self._content_is_abstract(schema_content), + schema_content, + ) - # 6. Validate traits (OP#13) - is_abstract = self._content_is_abstract(schema_content) - self._validate_traits(gts_id, is_abstract) + def validate_schema(self, gts_id: str) -> None: + """Validate a registered schema and all of its dependencies.""" + try: + schema_id = GtsID(gts_id) + except ValueError as error: + raise ValueError( + f"ID '{gts_id}' is not a schema (must end with '~')" + ) from error + if not schema_id.is_type: + raise ValueError(f"ID '{gts_id}' is not a schema (must end with '~')") + + schema_entity = self.get(schema_id.id) + if not schema_entity: + raise StoreGtsSchemaNotFound(schema_id.id) + if not schema_entity.is_schema: + raise ValueError(f"Entity '{schema_id.id}' is not a schema") + if not isinstance(schema_entity.content, dict): + raise ValueError( # noqa: TRY004 - keep ValueError for API compatibility + f"Schema '{schema_id.id}' content must be a dictionary" + ) + self.validate_schema_content(schema_id.id, schema_entity.content) + + def validate_instance_content(self, content: dict[str, Any], type_id: str) -> None: + """Validate unregistered instance content against a registered type schema.""" + schema_type = GtsID(type_id) + if not schema_type.is_type: + raise ValueError(f"ID '{type_id}' is not a schema (must end with '~')") + try: + schema = self.get_schema_content(schema_type.id) + except KeyError as error: + raise StoreGtsSchemaNotFound(schema_type.id) from error + + if isinstance(schema, dict) and self._content_is_abstract(schema): + raise ValueError( + f"type '{schema_type.id}' is abstract and cannot have direct instances" + ) + + schema_for_validation = _without_x_gts_ref(schema) + validator_class = validator_for(schema_for_validation) + validator = validator_class( + schema_for_validation, registry=self._create_reference_registry() + ) + validator.validate(content) + + x_gts_ref_validator = XGtsRefValidator(store=self) + x_gts_ref_errors = x_gts_ref_validator.validate_instance( + content, self._resolve_schema_refs(schema) + ) + if x_gts_ref_errors: + error_messages = [ + f"{err.field_path}: {err.reason}" for err in x_gts_ref_errors + ] + raise ValueError( + f"x-gts-ref validation failed: {'; '.join(error_messages)}" + ) def validate_instance( self, @@ -701,38 +782,11 @@ def validate_instance( raise StoreGtsObjectNotFound(gts_id) if not obj.type_id: raise StoreGtsSchemaForInstanceNotFound(lookup_id) - try: - schema = self.get_schema_content(obj.type_id) - except KeyError as e: - raise StoreGtsSchemaNotFound(obj.type_id) from e + if not isinstance(obj.content, dict): + raise TypeError(f"Instance '{lookup_id}' content must be a dictionary") logger.info(f"Validating instance {gts_id} against schema {obj.type_id}") - - # Check if the schema is abstract - abstract types cannot have direct instances - if isinstance(schema, dict) and self._content_is_abstract(schema): - raise ValueError( - f"type '{obj.type_id}' is abstract and cannot have direct instances" - ) - - schema_for_validation = _without_x_gts_ref(schema) - validator_class = validator_for(schema_for_validation) - validator = validator_class( - schema_for_validation, registry=self._create_reference_registry() - ) - validator.validate(obj.content) - - # Validate x-gts-ref constraints against the ref-resolved schema. - x_gts_ref_validator = XGtsRefValidator(store=self) - x_gts_ref_errors = x_gts_ref_validator.validate_instance( - obj.content, self._resolve_schema_refs(schema) - ) - if x_gts_ref_errors: - error_messages = [ - f"{err.field_path}: {err.reason}" for err in x_gts_ref_errors - ] - raise ValueError( - f"x-gts-ref validation failed: {'; '.join(error_messages)}" - ) + self.validate_instance_content(obj.content, obj.type_id) def cast( self, diff --git a/tests/test_ops.py b/tests/test_ops.py index 3e0fd90..0d30bbc 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -170,9 +170,7 @@ def test_match_true(self, ops): assert result.match is True def test_match_false(self, ops): - result = ops.match_id_pattern( - "gts.x.other._.foo.v1~", "gts.x.test.*" - ) + result = ops.match_id_pattern("gts.x.other._.foo.v1~", "gts.x.test.*") assert result.match is False def test_match_malformed_wildcard_candidate(self, ops): @@ -239,6 +237,73 @@ def test_validate_entity_invalid_id(self, ops): assert result.entity_type == "" +class TestValidateJson: + def test_validates_schema_without_storing_it(self, ops): + result = ops.validate_json(SCHEMA) + + assert result.ok is True + assert result.is_type_schema is True + assert ops.store.get(SCHEMA["$id"].removeprefix("gts://")) is None + + def test_validates_derived_schema_against_registered_parent(self, ops): + ops.add_entity(SCHEMA) + derived_schema = { + "$id": "gts://gts.x.test._.foo.v1~x.test._.bar.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "allOf": [{"$ref": "gts://gts.x.test._.foo.v1~"}], + } + + result = ops.validate_json(derived_schema) + + assert result.ok is True + assert ops.store.get("gts.x.test._.foo.v1~x.test._.bar.v1~") is None + + def test_validates_instance_against_explicit_type_without_storing_it(self, ops): + ops.add_entity(SCHEMA) + instance = {"name": "hi"} + + result = ops.validate_json(instance, explicit_type_id="gts.x.test._.foo.v1~") + + assert result.ok is True + assert result.type_id == "gts.x.test._.foo.v1~" + + def test_does_not_mutate_registry(self, ops, monkeypatch): + monkeypatch.setattr( + ops.store, + "register", + lambda entity: pytest.fail( + "transient validation must not register entities" + ), + ) + monkeypatch.setattr( + ops.store, + "unregister", + lambda entity_id: pytest.fail( + "transient validation must not unregister entities" + ), + ) + + result = ops.validate_json(SCHEMA) + + assert result.ok is True + + def test_rejects_missing_automatic_instance_type(self, ops): + result = ops.validate_json({"id": "gts.x.test._.missing_type.v1"}) + + assert result.ok is False + assert result.error == "Unable to determine instance type" + + def test_rejects_explicit_type_that_conflicts_with_body(self, ops): + result = ops.validate_json( + {"type": "gts.x.test._.other.v1~"}, + explicit_type_id="gts.x.test._.foo.v1~", + ) + + assert result.ok is False + assert "does not match path type" in result.error + + class TestSchemaGraphCompatibilityCast: def test_schema_graph(self, ops): ops.add_entity(SCHEMA) @@ -258,7 +323,9 @@ def test_cast_success(self, ops): assert result.error == "" def test_cast_error_wrapped(self, ops): - result = ops.cast("gts.x.test._.foo.v1~x.test._.missing.v1", "gts.x.test._.foo.v1~") + result = ops.cast( + "gts.x.test._.foo.v1~x.test._.missing.v1", "gts.x.test._.foo.v1~" + ) assert result.error != "" From 6304bdfc5a36650e0f466ae91e15ee1760f2d36c Mon Sep 17 00:00:00 2001 From: Artifizer Date: Fri, 11 Sep 2026 12:19:15 +0300 Subject: [PATCH 11/13] fix(cast): report compatibility for composed enum constraints Signed-off-by: Artifizer --- gts/src/gts/schema_cast.py | 24 +++++++++++++++++++++--- tests/test_schema_cast.py | 14 ++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/gts/src/gts/schema_cast.py b/gts/src/gts/schema_cast.py index ae42924..6b32e90 100644 --- a/gts/src/gts/schema_cast.py +++ b/gts/src/gts/schema_cast.py @@ -160,7 +160,7 @@ def cast( cls._validate_with_gts_id_tolerance(casted, to_schema_content, resolver) else: cls._validate_with_gts_id_tolerance(casted, to_schema_content, None) - is_fully_compatible = True + is_fully_compatible = is_backward and is_forward except js_exceptions.ValidationError as ve: reasons.append(ve.message) is_fully_compatible = False @@ -421,6 +421,15 @@ def _remove_gts_const_constraints(schema: Any) -> Any: return result + @staticmethod + def _flatten_property_schema(schema: dict[str, Any]) -> dict[str, Any]: + result: dict[str, Any] = {} + for sub_schema in schema.get("allOf", []): + if isinstance(sub_schema, dict): + result.update(GtsEntityCastResult._flatten_property_schema(sub_schema)) + result.update({key: value for key, value in schema.items() if key != "allOf"}) + return result + @staticmethod def _flatten_schema(schema: dict[str, Any]) -> dict[str, Any]: """Flatten a schema by merging allOf schemas.""" @@ -616,8 +625,12 @@ def _check_schema_compatibility( # Check properties that exist in both schemas common_props = set(old_props.keys()) & set(new_props.keys()) for prop in common_props: - old_prop_schema = old_props[prop] - new_prop_schema = new_props[prop] + old_prop_schema = GtsEntityCastResult._flatten_property_schema( + old_props[prop] + ) + new_prop_schema = GtsEntityCastResult._flatten_property_schema( + new_props[prop] + ) # Check if type changed old_type = old_prop_schema.get("type") @@ -647,6 +660,11 @@ def _check_schema_compatibility( errors.append( f"Property '{prop}' removed enum values: {removed_enum_values}" ) + elif old_enum: + if not check_backward: + errors.append(f"Property '{prop}' removed enum constraint") + elif new_enum and check_backward: + errors.append(f"Property '{prop}' added enum constraint") # Check constraint compatibility constraint_errors = GtsEntityCastResult._check_constraint_compatibility( diff --git a/tests/test_schema_cast.py b/tests/test_schema_cast.py index 09aa0c4..454cf08 100644 --- a/tests/test_schema_cast.py +++ b/tests/test_schema_cast.py @@ -342,6 +342,20 @@ def test_forward_removed_enum_values_flagged(self): assert not ok assert any("removed enum values" in e for e in errors) + def test_backward_added_enum_constraint_flagged(self): + old = {"properties": {"a": {"allOf": [{}]}}} + new = {"properties": {"a": {"allOf": [{"enum": ["x"]}]}}} + ok, errors = GtsEntityCastResult._check_backward_compatibility(old, new) + assert not ok + assert any("added enum constraint" in e for e in errors) + + def test_forward_removed_enum_constraint_flagged(self): + old = {"properties": {"a": {"allOf": [{"enum": ["x"]}]}}} + new = {"properties": {"a": {"allOf": [{}]}}} + ok, errors = GtsEntityCastResult._check_forward_compatibility(old, new) + assert not ok + assert any("removed enum constraint" in e for e in errors) + def test_nested_object_errors_prefixed(self): old = {"properties": {"a": {"type": "object", "properties": {"b": {"type": "string"}}}}} new = {"properties": {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}} From 1b6bd443253c322871b3305e26c0f938741689a8 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Fri, 11 Sep 2026 12:19:20 +0300 Subject: [PATCH 12/13] fix(traits): enforce formats and inherited requirements Signed-off-by: Artifizer --- gts/pyproject.toml | 2 +- gts/src/gts/derivation.py | 10 ++++++++++ gts/src/gts/traits.py | 20 +++++++++++++++----- gts/src/gts/x_gts_ref.py | 9 +++++++-- tests/test_traits.py | 20 ++++++++++++++++++++ 5 files changed, 53 insertions(+), 8 deletions(-) diff --git a/gts/pyproject.toml b/gts/pyproject.toml index 970fb5d..bc31a91 100644 --- a/gts/pyproject.toml +++ b/gts/pyproject.toml @@ -11,7 +11,7 @@ authors = [{ name = "GTS Community" }] license = { text = "Apache-2.0" } requires-python = ">=3.9" dependencies = [ - "jsonschema>=4.18,<5", + "jsonschema[format-nongpl]>=4.18,<5", "referencing>=0.30,<0.37", "jsonsubschema>=0.0.8,<0.1", "fastapi>=0.110,<1", diff --git a/gts/src/gts/derivation.py b/gts/src/gts/derivation.py index a3ce5a5..d59b64e 100644 --- a/gts/src/gts/derivation.py +++ b/gts/src/gts/derivation.py @@ -95,6 +95,16 @@ def _validate_derivation( f"closed constraint in base '{base_id}'" ) + base_required = set(base.get("required", [])) if isinstance(base, dict) else set() + derived_required = ( + set(derived.get("required", [])) if isinstance(derived, dict) else set() + ) + for name in sorted(base_required - derived_required): + errors.append( + f"property '{name}': derived schema '{derived_id}' does not require " + f"a property required by base '{base_id}'" + ) + # Admission fails closed: an unprovable inclusion is rejected. if check_accepted_set_inclusion(derived, base) is not True: errors.append( diff --git a/gts/src/gts/traits.py b/gts/src/gts/traits.py index bf4367d..2488680 100644 --- a/gts/src/gts/traits.py +++ b/gts/src/gts/traits.py @@ -19,6 +19,7 @@ import copy from typing import Any +from jsonschema import Draft7Validator, FormatChecker from jsonschema.validators import validator_for from . import derivation @@ -28,6 +29,8 @@ X_GTS_TRAITS = "x-gts-traits" MAX_RECURSION_DEPTH = 64 _MISSING = object() +_FORMAT_CHECKER = FormatChecker() +_FORMAT_CHECKER.checkers.update(Draft7Validator.FORMAT_CHECKER.checkers) class EffectiveTraits: @@ -51,7 +54,9 @@ def _has_schema(self) -> bool: def _has_explicit_values(self) -> bool: return isinstance(self.merged_traits, dict) and len(self.merged_traits) > 0 - def validate(self, check_unresolved: bool) -> list[str]: + def validate( + self, check_unresolved: bool, reference_store: Any | None = None + ) -> list[str]: """Return a list of error strings (empty means valid).""" errors = _validate_trait_schema_integrity(self.resolved_trait_schemas) if errors: @@ -76,7 +81,9 @@ def validate(self, check_unresolved: bool) -> list[str]: ] return [] - return _validate_trait_values(self.schema, self.values, check_unresolved) + return _validate_trait_values( + self.schema, self.values, check_unresolved, reference_store + ) # --- collection ------------------------------------------------------------ @@ -358,7 +365,7 @@ def _validate_traits_against_schema( try: cls = validator_for(validation_schema) - validator = cls(validation_schema) + validator = cls(validation_schema, format_checker=_FORMAT_CHECKER) for error in validator.iter_errors(effective_traits): errors.append(f"trait validation: {error.message}") except Exception as e: # noqa: BLE001 - surfaced as validation error message @@ -391,12 +398,15 @@ def _validate_traits_against_schema( def _validate_trait_values( - effective_traits_schema: Any, effective_traits: Any, check_unresolved: bool + effective_traits_schema: Any, + effective_traits: Any, + check_unresolved: bool, + reference_store: Any | None, ) -> list[str]: errors = _validate_traits_against_schema( effective_traits_schema, effective_traits, check_unresolved ) - xref = XGtsRefValidator() + xref = XGtsRefValidator(store=reference_store, require_registered_target=True) for err in xref.validate_instance(effective_traits, effective_traits_schema, ""): errors.append(f"trait x-gts-ref: {err.reason}") return errors diff --git a/gts/src/gts/x_gts_ref.py b/gts/src/gts/x_gts_ref.py index ddfc0ac..726258b 100644 --- a/gts/src/gts/x_gts_ref.py +++ b/gts/src/gts/x_gts_ref.py @@ -74,7 +74,9 @@ def __init__(self, field_path: str, value: Any, ref_pattern: str, reason: str): class XGtsRefValidator: """Validator for x-gts-ref constraints in GTS schemas.""" - def __init__(self, store: Any | None = None): + def __init__( + self, store: Any | None = None, require_registered_target: bool = False + ): """ Initialize validator. @@ -82,6 +84,7 @@ def __init__(self, store: Any | None = None): store: Optional GtsStore for resolving entity references """ self.store = store + self.require_registered_target = require_registered_target def validate_instance( self, instance: dict[str, Any], schema: dict[str, Any], instance_path: str = "" @@ -419,7 +422,9 @@ def _validate_gts_pattern( ) # Optionally check if entity exists in store - if self.store: + if self.store and ( + not self.require_registered_target or self.store.get(pattern) + ): entity = self.store.get(value) if not entity: return XGtsRefValidationError( diff --git a/tests/test_traits.py b/tests/test_traits.py index 576389f..72a8342 100644 --- a/tests/test_traits.py +++ b/tests/test_traits.py @@ -157,6 +157,26 @@ def test_valid_trait_values_pass(self): effective = build_effective_traits([schema], {"a": "hi"}, None) assert effective.validate(check_unresolved=True) == [] + def test_standard_trait_formats_are_enforced(self): + schema = { + "type": "object", + "properties": { + "email": {"type": "string", "format": "email"}, + "time": {"type": "string", "format": "time"}, + }, + } + + assert ( + build_effective_traits( + [schema], {"email": "user@example.com", "time": "10:30:00Z"}, None + ).validate(check_unresolved=True) + == [] + ) + errors = build_effective_traits( + [schema], {"email": "not-an-email", "time": "10:30:00Z"}, None + ).validate(check_unresolved=True) + assert any("is not a 'email'" in error for error in errors) + def test_invalid_trait_type_fails(self): schema = { "type": "object", From 3acc7d449db10fdb0e806a4420286790f87200aa Mon Sep 17 00:00:00 2001 From: Artifizer Date: Fri, 11 Sep 2026 12:30:11 +0300 Subject: [PATCH 13/13] chore: update PY_ENV_PYTHON in Makefile for Windows Signed-off-by: Artifizer --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1ac5666..dc588e2 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ SHELL := /bin/bash PYTHON_BOOTSTRAP ?= $(shell command -v python3 2>/dev/null || command -v python 2>/dev/null || echo python3) PY_ENV_DIR ?= .venv ifeq ($(OS),Windows_NT) -PY_ENV_PYTHON := $(PY_ENV_DIR)/Scripts/python +PY_ENV_PYTHON := $(PY_ENV_DIR)/Scripts/python.exe else PY_ENV_PYTHON := $(PY_ENV_DIR)/bin/python endif