diff --git a/Makefile b/Makefile index f9f3f1b..dc588e2 100644 --- a/Makefile +++ b/Makefile @@ -11,10 +11,11 @@ SHELL := /bin/bash PYTHON_BOOTSTRAP ?= $(shell command -v python3 2>/dev/null || command -v python 2>/dev/null || echo python3) PY_ENV_DIR ?= .venv ifeq ($(OS),Windows_NT) -PYTHON ?= $(PY_ENV_DIR)/Scripts/python +PY_ENV_PYTHON := $(PY_ENV_DIR)/Scripts/python.exe else -PYTHON ?= $(PY_ENV_DIR)/bin/python +PY_ENV_PYTHON := $(PY_ENV_DIR)/bin/python endif +PYTHON ?= $(PY_ENV_PYTHON) PY_ENV_STAMP := $(PY_ENV_DIR)/.stamp INSTALL_STAMP := $(PY_ENV_DIR)/.install-stamp LOCAL_DIST_DIR := dist-install-local @@ -39,10 +40,10 @@ help: # Create/update the virtual environment and install dev/test dependencies py-env: $(PY_ENV_STAMP) -$(PY_ENV_DIR)/bin/python: - $(PYTHON_BOOTSTRAP) -m venv $(PY_ENV_DIR) +$(PY_ENV_PYTHON): + $(PYTHON_BOOTSTRAP) -m venv --clear $(PY_ENV_DIR) -$(PY_ENV_STAMP): gts/pyproject.toml .gts-spec/tests/requirements.txt Makefile +$(PY_ENV_STAMP): $(PY_ENV_PYTHON) gts/pyproject.toml .gts-spec/tests/requirements.txt Makefile @echo "Creating/updating Python virtual environment in $(PY_ENV_DIR)..." $(PYTHON_BOOTSTRAP) -m venv $(PY_ENV_DIR) $(PYTHON) -m pip install --upgrade pip @@ -64,7 +65,7 @@ build: py-env $(PYTHON) -m build --outdir dist ./gts # Install the locally built wheel, equivalent to installing the published gts package -install-local: $(if $(filter $(PY_ENV_DIR)/bin/python,$(PYTHON)),$(PY_ENV_DIR)/bin/python) +install-local: $(if $(filter $(PY_ENV_PYTHON),$(PYTHON)),$(PY_ENV_PYTHON)) @rm -rf $(LOCAL_DIST_DIR) $(PYTHON) -m pip install --upgrade build $(PYTHON) -m build --outdir $(LOCAL_DIST_DIR) ./gts diff --git a/README.md b/README.md index 594c9f9..45272f0 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and JSON/JSON Schema artifacts. -## Roadmap +Current supported GTS spec version: `0.13.1` -Current supported GTS spec version: 0.13 +## Roadmap Featureset: diff --git a/gts/openapi.json b/gts/openapi.json index df00e6f..69dc264 100644 --- a/gts/openapi.json +++ b/gts/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "GTS Server", - "version": "0.13.0" + "version": "0.13.1" }, "paths": { "/entities": { diff --git a/gts/pyproject.toml b/gts/pyproject.toml index 893c599..bc31a91 100644 --- a/gts/pyproject.toml +++ b/gts/pyproject.toml @@ -4,14 +4,14 @@ build-backend = "hatchling.build" [project] name = "gts" -version = "0.13.0" +version = "0.13.1" description = "Global Type System (GTS) helpers: identifiers, parsing, validation, and operations" readme = "README.md" authors = [{ name = "GTS Community" }] license = { text = "Apache-2.0" } requires-python = ">=3.9" dependencies = [ - "jsonschema>=4.18,<5", + "jsonschema[format-nongpl]>=4.18,<5", "referencing>=0.30,<0.37", "jsonsubschema>=0.0.8,<0.1", "fastapi>=0.110,<1", diff --git a/gts/src/gts/_cli.py b/gts/src/gts/_cli.py index fc5cd1b..db7dd34 100644 --- a/gts/src/gts/_cli.py +++ b/gts/src/gts/_cli.py @@ -5,6 +5,7 @@ import logging import sys +from ._json_validation import GtsJsonValidator from ._server import GtsHttpServer from .ops import GtsOps @@ -18,6 +19,14 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument( "--path", help="Path to json and schema files or directories (global default)" ) + p.add_argument( + "--exclude", + default="node_modules,dist,build,.git,target", + help=( + "Comma-separated directory names to exclude when scanning " + "(default: node_modules,dist,build,.git,target)" + ), + ) sub = p.add_subparsers(dest="op", required=True) s = sub.add_parser("validate-id", help="Validate a GTS ID format") @@ -34,6 +43,11 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--gts-id", required=True) s.add_argument("--scope", choices=["major", "full"], default="major") + s = sub.add_parser( + "validate-all", help="Validate all JSON documents in a file or directory" + ) + s.add_argument("--path", dest="scan_path", help="JSON file or directory to scan") + s = sub.add_parser( "validate-instance", help="Validate an instance against its schema" ) @@ -116,8 +130,16 @@ def main(argv: list[str] | None = None) -> None: ) try: + # Parse the comma-separated --exclude option into a list of dir names + exclude = [e.strip() for e in (args.exclude or "").split(",") if e.strip()] + # Helper to create GtsOps with common arguments - ops = GtsOps(path=args.path, config=args.config, verbose=args.verbose) + ops = GtsOps( + path=args.path, + config=args.config, + verbose=args.verbose, + exclude=exclude, + ) if args.op == "server": server = GtsHttpServer(ops=ops) @@ -146,6 +168,17 @@ def main(argv: list[str] | None = None) -> None: json.dump(out, sys.stdout, ensure_ascii=False, indent=2) sys.stdout.write("\n") return + elif args.op == "validate-all": + scan_path = args.scan_path or args.path + if not scan_path: + parser.error("validate-all requires --path") + result = GtsJsonValidator(scan_path, ops.cfg, exclude=exclude).validate() + out = result.to_dict() + json.dump(out, sys.stdout, ensure_ascii=False, indent=2) + sys.stdout.write("\n") + if not result.ok: + raise SystemExit(1) + return elif args.op == "validate-id": out = ops.validate_id(args.gts_id).to_dict() elif args.op == "parse-id": diff --git a/gts/src/gts/_json_validation.py b/gts/src/gts/_json_validation.py new file mode 100644 index 0000000..46104b2 --- /dev/null +++ b/gts/src/gts/_json_validation.py @@ -0,0 +1,322 @@ +from __future__ import annotations + +import json +import os +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from .entities import GtsEntity, GtsFile +from .files_reader import DEFAULT_EXCLUDE_LIST +from .gts import GTS_PREFIX, GTS_URI_PREFIX, GtsID +from .store import GtsStore + +_X_GTS_REF_KEYWORD = "x-gts-ref" + + +@dataclass +class GtsJsonValidationIssue: + file: str + stage: str + message: str + index: int | None = None + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "file": self.file, + "stage": self.stage, + "message": self.message, + } + if self.index is not None: + result["index"] = self.index + return result + + +@dataclass +class GtsJsonValidationResult: + files: int = 0 + documents: int = 0 + gts_entities: int = 0 + schemas: int = 0 + instances: int = 0 + issues: list[GtsJsonValidationIssue] = field(default_factory=list) + + @property + def ok(self) -> bool: + return not self.issues + + def to_dict(self) -> dict[str, Any]: + return { + "ok": self.ok, + "files": self.files, + "documents": self.documents, + "gts_entities": self.gts_entities, + "schemas": self.schemas, + "instances": self.instances, + "issues": [issue.to_dict() for issue in self.issues], + } + + +class GtsJsonValidator: + def __init__(self, path: str, cfg: Any, exclude: list[str] | None = None) -> None: + self.path = Path(path).expanduser() + self.cfg = cfg + self.exclude = list(exclude) if exclude else list(DEFAULT_EXCLUDE_LIST) + self.result = GtsJsonValidationResult() + self.entities: list[GtsEntity] = [] + + def validate(self) -> GtsJsonValidationResult: + for file_path in self._json_files(): + self._read_file(file_path) + self._check_schema_field_type() + store = self._register_gts_entities() + schemas_count, instances_count = self._count_schema_instance() + self.result.schemas = schemas_count + self.result.instances = instances_count + self.result.gts_entities = schemas_count + instances_count + self._validate_schemas(store) + self._validate_instances(store) + return self.result + + def _json_files(self) -> list[Path]: + resolved = self.path.resolve(strict=False) + if resolved.is_file(): + if resolved.suffix.lower() == ".json": + return [resolved] + self._issue(resolved, "discovery", "Expected a .json file") + return [] + if not resolved.is_dir(): + self._issue( + resolved, "discovery", "Path does not exist or is not accessible" + ) + return [] + + files: list[Path] = [] + seen_files: set[Path] = set() + seen_dirs: set[tuple[int, int]] = set() + walk_errors: list[OSError] = [] + + def _on_walk_error(err: OSError) -> None: + walk_errors.append(err) + + for root, dirs, names in os.walk( + resolved, followlinks=True, onerror=_on_walk_error + ): + # Prevent symlink cycles by tracking visited directory identities + root_stat = os.stat(root) + dir_id = (root_stat.st_dev, root_stat.st_ino) + if dir_id in seen_dirs: + dirs.clear() + continue + seen_dirs.add(dir_id) + + # Prune excluded directories (defaults to DEFAULT_EXCLUDE_LIST, + # overridable via the CLI --exclude option) + dirs[:] = [d for d in dirs if d not in self.exclude] + + for name in names: + if Path(name).suffix.lower() == ".json": + rp = Path(root, name).resolve(strict=False) + if rp not in seen_files: + seen_files.add(rp) + files.append(rp) + + for err in walk_errors: + self._issue(resolved, "discovery", f"Traversal error: {err}") + + return sorted(files) + + @staticmethod + def _is_gts_marker(text: str) -> bool: + return ( + GTS_PREFIX in text or GTS_URI_PREFIX in text or _X_GTS_REF_KEYWORD in text + ) + + def _read_file(self, file_path: Path) -> None: + try: + content_str = file_path.read_text(encoding="utf-8") + except Exception as error: # noqa: BLE001 - report this document and continue + self._issue(file_path, "json", str(error)) + return + + if not self._is_gts_marker(content_str): + return + self.result.files += 1 + + try: + content = json.loads(content_str) + except Exception as error: # noqa: BLE001 - report this document and continue + self._issue(file_path, "json", str(error)) + return + + values = content if isinstance(content, list) else [content] + file = GtsFile(path=str(file_path), name=file_path.name, content=content) + for index, value in enumerate(values): + self.result.documents += 1 + self.entities.append( + GtsEntity( + file=file, + list_sequence=index if isinstance(content, list) else None, + content=value, + cfg=self.cfg, + ) + ) + + def _check_schema_field_type(self) -> None: + for entity in self.entities: + content = entity.content + if not isinstance(content, dict): + continue + schema_val = content.get("$schema") + if schema_val is not None and not isinstance(schema_val, str): + self._issue(entity, "json-schema", "$schema must be a string") + + def _register_gts_entities(self) -> GtsStore: + store = GtsStore(reader=None) # type: ignore[arg-type] + keys: set[str] = set() + for entity in self.entities: + if not self._is_gts_related(entity.content): + continue + key = self._registry_key(entity) + if key is None: + if entity.is_schema: + # A schema with a plain gts. $id (not gts://) yields no + # gts_id at the core entity layer, so it lands here. + self._issue( + entity, + "registry", + "GTS schema has a malformed or non-GTS $id", + ) + continue + if not entity.is_schema and entity.selected_entity_field is None: + raw_id = entity.raw_id + if raw_id is not None: + entity.raw_id = str(uuid.uuid5(uuid.NAMESPACE_URL, raw_id)) + key = entity.raw_id + if key in keys: + self._issue(entity, "registry", f"Duplicate GTS entity ID '{key}'") + continue + keys.add(key) + store.register(entity) + return store + + def _count_schema_instance(self) -> tuple[int, int]: + schemas = 0 + instances = 0 + for entity in self.entities: + if self._registry_key(entity) is None: + continue + if entity.is_schema: + schemas += 1 + else: + instances += 1 + return schemas, instances + + def _is_gts_related(self, value: Any) -> bool: + if not isinstance(value, dict): + return False + # Check configured identifier fields for GTS IDs. + # Accept valid IDs and also detect likely-but-malformed ones + # (gts:// or gts. prefix) so they get diagnosed during registration + # rather than silently skipped. + for f in self.cfg.entity_id_fields: + v = value.get(f) + if isinstance(v, str) and self._looks_gts(v): + return True + for f in self.cfg.schema_id_fields: + v = value.get(f) + if isinstance(v, str) and self._looks_gts(v): + return True + return False + + @staticmethod + def _looks_gts(v: str) -> bool: + normalized = v.removeprefix(GTS_URI_PREFIX) + return normalized.startswith(GTS_PREFIX) or v.startswith(GTS_URI_PREFIX) + + @staticmethod + def _registry_key(entity: GtsEntity) -> str | None: + if entity.is_schema and entity.gts_id: + return entity.gts_id.id + if ( + not entity.is_schema + and entity.raw_id + and ( + entity.gts_id + or (entity.type_id is not None and GtsID.is_valid(entity.type_id)) + ) + ): + return entity.raw_id + return None + + def _validate_schemas(self, store: GtsStore) -> None: + pending: list[tuple[int, str, str, int | None, GtsEntity]] = [] + for entity in self.entities: + if not entity.is_schema or not entity.gts_id: + continue + gid = entity.gts_id + if store.get(gid.id) is not entity: + continue + depth = len(gid.gts_id_segments) + file = entity.file.path if entity.file else entity.label + pending.append((depth, gid.id, file, entity.list_sequence, entity)) + pending.sort(key=lambda t: (t[0], t[1], t[2], t[3] if t[3] is not None else -1)) + + for depth, _gts_id, _file, _idx, entity in pending: + stage = "base-type" if depth <= 1 else "derived-type" + try: + store.validate_schema(entity.gts_id.id) # type: ignore[union-attr] + except Exception as error: # noqa: BLE001 - report this document and continue + self._issue(entity, stage, str(error)) + + @staticmethod + def _schema_depth(entity: GtsEntity) -> int: + return len(entity.gts_id.gts_id_segments) if entity.gts_id else 0 + + @staticmethod + def _entity_depth(entity: GtsEntity) -> int: + if entity.gts_id: + return len(entity.gts_id.gts_id_segments) + if entity.type_id and GtsID.is_valid(entity.type_id): + return len(GtsID(entity.type_id).gts_id_segments) + return 0 + + def _validate_instances(self, store: GtsStore) -> None: + pending: list[tuple[int, str, str, int | None, str, GtsEntity]] = [] + for entity in self.entities: + if entity.is_schema: + continue + key = self._registry_key(entity) + if key is None: + continue + # Skip rejected duplicates: only validate the registered entity + if store.get(key) is not entity: + continue + depth = self._entity_depth(entity) + gts_id_str = entity.gts_id.id if entity.gts_id else "" + file = entity.file.path if entity.file else entity.label + pending.append((depth, gts_id_str, file, entity.list_sequence, key, entity)) + pending.sort(key=lambda t: (t[0], t[1], t[2], t[3] if t[3] is not None else -1)) + + for _depth, _gts_id, _file, _idx, registry_key, entity in pending: + try: + store.validate_instance(registry_key) + except Exception as error: # noqa: BLE001 - report this document and continue + self._issue(entity, "instance", str(error)) + + def _issue( + self, + source: Path | GtsEntity, + stage: str, + message: str, + ) -> None: + if isinstance(source, GtsEntity): + file = source.file.path if source.file else source.label + index = source.list_sequence + else: + file = str(source) + index = None + self.result.issues.append( + GtsJsonValidationIssue(file=file, stage=stage, message=message, index=index) + ) diff --git a/gts/src/gts/_server.py b/gts/src/gts/_server.py index 653e65b..89f4bf1 100644 --- a/gts/src/gts/_server.py +++ b/gts/src/gts/_server.py @@ -41,7 +41,9 @@ def __init__(self, app: FastAPI, verbose: int) -> None: async def dispatch(self, request, call_next): if not self.verbose: - return await call_next(request) + response = await call_next(request) + response.headers["connection"] = "close" + return response start = time.time() @@ -60,6 +62,7 @@ async def receive(): request = Request(request.scope, receive) response = await call_next(request) + response.headers["connection"] = "close" dur = (time.time() - start) * 1000.0 # Determine status color @@ -185,7 +188,7 @@ def __init__( self.host = host self.port = port self.base_url = f"http://{self.host}:{self.port}" - self.app = FastAPI(title="GTS Server", version="0.13.0") + self.app = FastAPI(title="GTS Server", version="0.13.1") self.app.add_middleware( _RequestLoggingMiddleware, verbose=self.ops.verbose, @@ -275,6 +278,18 @@ def _register_routes(self) -> None: methods=["POST"], summary="Validate instance by GTS ID", ) + app.add_api_route( + "/validate-json", + self.validate_json, + methods=["POST"], + summary="Validate unregistered JSON entity", + ) + app.add_api_route( + "/validate-json/{gts_type:path}", + self.validate_json_as_type, + methods=["POST"], + summary="Validate unregistered JSON instance against a type", + ) # Op #12 - validate type schema app.add_api_route( "/validate-type-schema", @@ -371,6 +386,19 @@ async def id_to_uuid(self, id: str = Query(..., alias="gts_id")) -> dict[str, An async def validate_instance(self, body: ValidateInstanceRequest) -> dict[str, Any]: return self.ops.validate_instance(body.instance_id).to_dict() + async def validate_json( + self, + body: dict[str, Any] = Body(...), # noqa: B008 - FastAPI dependency pattern + ) -> dict[str, Any]: + return self.ops.validate_json(body).to_dict() + + async def validate_json_as_type( + self, + gts_type: str, + body: dict[str, Any] = Body(...), # noqa: B008 - FastAPI dependency pattern + ) -> dict[str, Any]: + return self.ops.validate_json(body, explicit_type_id=gts_type).to_dict() + async def validate_type_schema( self, body: ValidateTypeSchemaRequest ) -> dict[str, Any]: diff --git a/gts/src/gts/derivation.py b/gts/src/gts/derivation.py index a3ce5a5..d59b64e 100644 --- a/gts/src/gts/derivation.py +++ b/gts/src/gts/derivation.py @@ -95,6 +95,16 @@ def _validate_derivation( f"closed constraint in base '{base_id}'" ) + base_required = set(base.get("required", [])) if isinstance(base, dict) else set() + derived_required = ( + set(derived.get("required", [])) if isinstance(derived, dict) else set() + ) + for name in sorted(base_required - derived_required): + errors.append( + f"property '{name}': derived schema '{derived_id}' does not require " + f"a property required by base '{base_id}'" + ) + # Admission fails closed: an unprovable inclusion is rejected. if check_accepted_set_inclusion(derived, base) is not True: errors.append( diff --git a/gts/src/gts/entities.py b/gts/src/gts/entities.py index 4c92028..171e68e 100644 --- a/gts/src/gts/entities.py +++ b/gts/src/gts/entities.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any -from .gts import GtsID +from .gts import GTS_PREFIX, GTS_URI_PREFIX, GtsID from .schema_cast import GtsEntityCastResult, SchemaCastError if TYPE_CHECKING: @@ -130,7 +130,15 @@ def __init__( self.type_id and GtsID.is_valid(self.type_id) ): idv = self.type_id - self.gts_id = GtsID(idv) if idv and GtsID.is_valid(idv) else None + # Enforce gts:// URI form for schema $id at the core layer. Per + # gts-spec, a schema must place its identifier in $id as a gts:// + # URI, not the bare gts. prefix. Leaving gts_id as None makes every + # client (server, batch validator, direct callers) reject it + # uniformly, mirroring gts-go extract.go and gts-rust entities.rs. + if self.is_schema and self._schema_id_uses_plain_prefix(): + self.gts_id = None + else: + self.gts_id = GtsID(idv) if idv and GtsID.is_valid(idv) else None # Set label if self.file and self.list_sequence is not None: @@ -287,6 +295,21 @@ def _get_field_value(self, field: str) -> str | None: return v return None + def _schema_id_uses_plain_prefix(self) -> bool: + """Return True if the raw schema $id uses the bare gts. prefix. + + Schemas must express their identifier as a gts:// URI in $id. A raw + $id that starts with the plain gts. prefix (without gts://) is invalid + and must not yield a gts_id. + """ + if not isinstance(self.content, dict): + return False + raw = self.content.get("$id") + if not isinstance(raw, str): + return False + raw = raw.strip() + return raw.startswith(GTS_PREFIX) and not raw.startswith(GTS_URI_PREFIX) + def _first_non_empty_field(self, fields: list[str]) -> tuple[str, str] | None: """Find first non-empty field value in order. diff --git a/gts/src/gts/files_reader.py b/gts/src/gts/files_reader.py index 66cd24d..f8ca43e 100644 --- a/gts/src/gts/files_reader.py +++ b/gts/src/gts/files_reader.py @@ -12,7 +12,9 @@ from .entities import DEFAULT_GTS_CONFIG, GtsConfig, GtsEntity, GtsFile from .store import GtsReader -EXCLUDE_LIST = ["node_modules", "dist", "build"] +# Default directory names skipped during recursive scanning. The CLI --exclude +# option overrides this per invocation. +DEFAULT_EXCLUDE_LIST = ["node_modules", "dist", "build", ".git", "target"] logger = logging.getLogger(__name__) @@ -20,13 +22,20 @@ class GtsFileReader(GtsReader): """Reads GTS entities from JSON and YAML files in directories specified by path.""" - def __init__(self, path: str | list[str], cfg: GtsConfig | None = None) -> None: + def __init__( + self, + path: str | list[str], + cfg: GtsConfig | None = None, + exclude: list[str] | None = None, + ) -> None: """ Initialize FileReader with one or more paths. Args: path: Single path string or list of paths (files or directories) cfg: GtsConfig for entity ID extraction (defaults to DEFAULT_GTS_CONFIG) + exclude: Directory names to skip while scanning (defaults to + DEFAULT_EXCLUDE_LIST) """ self.paths: list[Path] = [] if isinstance(path, str): @@ -35,6 +44,7 @@ def __init__(self, path: str | list[str], cfg: GtsConfig | None = None) -> None: self.paths = [Path(os.path.expanduser(p)) for p in path] self.cfg = cfg or DEFAULT_GTS_CONFIG + self.exclude = list(exclude) if exclude else list(DEFAULT_EXCLUDE_LIST) self._files: list[Path] = [] self._current_index = 0 self._current_file_entities: list[GtsEntity] = [] @@ -45,6 +55,7 @@ def _collect_files(self) -> None: """Collect all JSON and YAML files from the specified paths, following symlinks.""" valid_extensions = {".json", ".jsonc", ".gts", ".yaml", ".yml"} seen: set[str] = set() + seen_dirs: set[tuple[int, int]] = set() collected: list[Path] = [] for path in self.paths: @@ -61,9 +72,15 @@ def _collect_files(self) -> None: elif resolved_path.is_dir(): # Recursively scan for all valid file types, following symlinks for root, dirs, files in os.walk(resolved_path, followlinks=True): - for exclude in EXCLUDE_LIST: - if exclude in dirs: - dirs.remove(exclude) + # Prevent symlink cycles by tracking visited directory identities + root_stat = os.stat(root) + dir_id = (root_stat.st_dev, root_stat.st_ino) + if dir_id in seen_dirs: + dirs.clear() + continue + seen_dirs.add(dir_id) + + dirs[:] = [d for d in dirs if d not in self.exclude] for fname in files: ext = os.path.splitext(fname)[1].lower() if ext in valid_extensions: diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index 4954c83..8ae746c 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -152,6 +152,30 @@ def to_dict(self) -> dict[str, Any]: return result +@dataclass +class GtsJsonValidationResult: + """Result of validating an unregistered JSON entity.""" + + ok: bool + id: str = "" + type_id: str | None = None + is_type_schema: bool = False + error: str = "" + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "ok": self.ok, + "is_type_schema": self.is_type_schema, + } + if self.id: + result["id"] = self.id + if self.type_id: + result["type_id"] = self.type_id + if self.error: + result["error"] = self.error + return result + + @dataclass class GtsSchemaGraphResult: """Result of building a schema graph for an entity.""" @@ -297,11 +321,17 @@ def __init__( path: str | builtins.list[str] | None = None, config: str | None = None, verbose: int = 0, + exclude: builtins.list[str] | None = None, ) -> None: self.verbose = verbose self.cfg = self._load_config(config) self.path: str | list[str] | None = path - self._reader = GtsFileReader(self.path, cfg=self.cfg) if self.path else None + self.exclude = exclude + self._reader = ( + GtsFileReader(self.path, cfg=self.cfg, exclude=self.exclude) + if self.path + else None + ) self.store = GtsStore(self._reader) if self._reader else GtsStore(reader=None) # type: ignore[arg-type] @staticmethod @@ -345,7 +375,7 @@ def _load_config(self, config_path: str | None) -> GtsConfig: def reload_from_path(self, path: str | builtins.list[str]) -> None: self.path = path - self._reader = GtsFileReader(self.path, cfg=self.cfg) + self._reader = GtsFileReader(self.path, cfg=self.cfg, exclude=self.exclude) self.store = GtsStore(self._reader) def add_entity( @@ -360,31 +390,21 @@ def add_entity( ): return GtsAddEntityResult( ok=False, - error="Instance must have an id field", + error="Unable to detect GTS ID in instance entity: Instance must have an id field", is_type_schema=False, ) - # Schemas MUST have a valid GTS ID + # Schemas MUST have a valid GTS ID. The core entity layer already + # rejects a plain gts. $id (without gts://) by leaving gts_id as None, + # so this single guard covers both the malformed and the wrong-prefix + # cases uniformly for every client. if entity.is_schema and not entity.gts_id: return GtsAddEntityResult( - ok=False, error="Unable to detect GTS ID in schema" + ok=False, + error="Unable to detect GTS ID in schema", + is_type_schema=entity.is_schema, ) - # Validate $id prefix for schemas: must use gts:// URI, not plain gts. - if entity.is_schema and validate: - raw_id = content.get("$id", "") - # Reject plain gts. prefix (without gts://) - if ( - isinstance(raw_id, str) - and raw_id.startswith("gts.") - and not raw_id.startswith("gts://") - ): - return GtsAddEntityResult( - ok=False, - error="Schema $id must use gts:// URI format, not plain gts. prefix", - is_type_schema=True, - ) - store_key = entity.gts_id.id if entity.is_schema else entity.raw_id previous = self.store.get(store_key) self.store.register(entity) @@ -528,6 +548,83 @@ def uuid(self, gts_id: str) -> GtsUuidResult: g = GtsID(gts_id) return GtsUuidResult(id=g.id, uuid=str(g.to_uuid())) + def validate_json( + self, content: dict[str, Any], explicit_type_id: str | None = None + ) -> GtsJsonValidationResult: + """Validate JSON without retaining it in the registry.""" + entity = GtsEntity(content=content, cfg=self.cfg) + if explicit_type_id is not None: + try: + explicit_type = GtsID(explicit_type_id) + except ValueError: + if explicit_type_id.startswith(("gts.", "gts://")): + return GtsJsonValidationResult( + ok=False, + error=f"Explicit type '{explicit_type_id}' must be GTS Type schema", + ) + return GtsJsonValidationResult( + ok=False, error=f"Invalid GTS Type Schema ID: {explicit_type_id}" + ) + if not explicit_type.is_type: + return GtsJsonValidationResult( + ok=False, + error=f"Explicit type '{explicit_type_id}' must be GTS Type schema", + ) + explicit_type_id = explicit_type.id + if entity.is_schema: + return GtsJsonValidationResult( + ok=False, + is_type_schema=True, + error="Explicit type validation only accepts instance JSON", + ) + if entity.type_id and entity.type_id != explicit_type_id: + return GtsJsonValidationResult( + ok=False, + type_id=explicit_type_id, + error=( + f"Declared type '{entity.type_id}' does not match path type " + f"'{explicit_type_id}'" + ), + ) + entity.type_id = explicit_type_id + elif entity.is_schema and not entity.gts_id: + return GtsJsonValidationResult( + ok=False, + is_type_schema=True, + error="Unable to detect GTS ID in schema", + ) + elif not entity.is_schema and not entity.type_id: + return GtsJsonValidationResult( + ok=False, + error="Unable to determine instance type", + ) + + try: + if entity.is_schema: + self.store.validate_schema_content(entity.gts_id.id, content) # type: ignore[union-attr] + else: + self.store.validate_instance_content(content, entity.type_id) + except Exception as error: # noqa: BLE001 - converted to a result object at API boundary + error_message = str(error) + if entity.is_schema and "not found for chain validation" in error_message: + error_message = f"Parent GTS Type Schema not found: {error_message}" + elif explicit_type_id is not None and "not found in store" in error_message: + error_message = f"GTS Type Schema not found: {explicit_type_id}" + return GtsJsonValidationResult( + ok=False, + id=entity.gts_id.id if entity.gts_id else (entity.raw_id or ""), + type_id=entity.type_id, + is_type_schema=entity.is_schema, + error=error_message, + ) + + return GtsJsonValidationResult( + ok=True, + id=entity.gts_id.id if entity.gts_id else (entity.raw_id or ""), + type_id=entity.type_id, + is_type_schema=entity.is_schema, + ) + def validate_instance(self, gts_id: str) -> GtsValidationResult: try: self.store.validate_instance(gts_id) diff --git a/gts/src/gts/schema_cast.py b/gts/src/gts/schema_cast.py index ae42924..6b32e90 100644 --- a/gts/src/gts/schema_cast.py +++ b/gts/src/gts/schema_cast.py @@ -160,7 +160,7 @@ def cast( cls._validate_with_gts_id_tolerance(casted, to_schema_content, resolver) else: cls._validate_with_gts_id_tolerance(casted, to_schema_content, None) - is_fully_compatible = True + is_fully_compatible = is_backward and is_forward except js_exceptions.ValidationError as ve: reasons.append(ve.message) is_fully_compatible = False @@ -421,6 +421,15 @@ def _remove_gts_const_constraints(schema: Any) -> Any: return result + @staticmethod + def _flatten_property_schema(schema: dict[str, Any]) -> dict[str, Any]: + result: dict[str, Any] = {} + for sub_schema in schema.get("allOf", []): + if isinstance(sub_schema, dict): + result.update(GtsEntityCastResult._flatten_property_schema(sub_schema)) + result.update({key: value for key, value in schema.items() if key != "allOf"}) + return result + @staticmethod def _flatten_schema(schema: dict[str, Any]) -> dict[str, Any]: """Flatten a schema by merging allOf schemas.""" @@ -616,8 +625,12 @@ def _check_schema_compatibility( # Check properties that exist in both schemas common_props = set(old_props.keys()) & set(new_props.keys()) for prop in common_props: - old_prop_schema = old_props[prop] - new_prop_schema = new_props[prop] + old_prop_schema = GtsEntityCastResult._flatten_property_schema( + old_props[prop] + ) + new_prop_schema = GtsEntityCastResult._flatten_property_schema( + new_props[prop] + ) # Check if type changed old_type = old_prop_schema.get("type") @@ -647,6 +660,11 @@ def _check_schema_compatibility( errors.append( f"Property '{prop}' removed enum values: {removed_enum_values}" ) + elif old_enum: + if not check_backward: + errors.append(f"Property '{prop}' removed enum constraint") + elif new_enum and check_backward: + errors.append(f"Property '{prop}' added enum constraint") # Check constraint compatibility constraint_errors = GtsEntityCastResult._check_constraint_compatibility( diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 7ff55b7..72edaa3 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -302,11 +302,16 @@ def _validate_schema_x_gts_refs(self, gts_id: str) -> None: if not schema_entity.is_schema: raise ValueError(f"Entity '{gts_id}' is not a schema") + self._validate_schema_x_gts_refs_content(gts_id, schema_entity.content) + + def _validate_schema_x_gts_refs_content( + self, gts_id: str, schema_content: dict[str, Any] + ) -> None: logger.info(f"Validating schema x-gts-ref fields for {gts_id}") # Validate x-gts-ref constraints in the schema x_gts_ref_validator = XGtsRefValidator(store=self) - x_gts_ref_errors = x_gts_ref_validator.validate_schema(schema_entity.content) + x_gts_ref_errors = x_gts_ref_validator.validate_schema(schema_content) if x_gts_ref_errors: error_messages = [ f"{err.field_path}: {err.reason}" for err in x_gts_ref_errors @@ -317,7 +322,15 @@ def _validate_schema_x_gts_refs(self, gts_id: str) -> None: @staticmethod def _validate_gts_keywords(content: dict[str, Any]) -> None: - """Validate x-gts-final, x-gts-abstract, x-gts-traits, x-gts-traits-schema placement.""" + """Validate supported GTS extensions and their placement.""" + + top_level_keywords = { + "x-gts-final", + "x-gts-abstract", + "x-gts-traits", + "x-gts-traits-schema", + } + supported_keywords = top_level_keywords | {"x-gts-ref"} def _contains_key_recursive(value: Any, key: str) -> bool: if isinstance(value, dict): @@ -348,14 +361,20 @@ def _contains_key_recursive(value: Any, key: str) -> bool: "schema cannot declare both x-gts-final and x-gts-abstract as true" ) + def _validate_extensions(value: Any) -> None: + if isinstance(value, dict): + for key, nested_value in value.items(): + if key.startswith("x-gts-") and key not in supported_keywords: + raise ValueError(f"Unsupported GTS extension keyword: {key}") + _validate_extensions(nested_value) + elif isinstance(value, list): + for item in value: + _validate_extensions(item) + + _validate_extensions(content) + # Check that x-gts-final/x-gts-abstract/x-gts-traits/x-gts-traits-schema # appear only at the top level - top_level_keywords = { - "x-gts-final", - "x-gts-abstract", - "x-gts-traits", - "x-gts-traits-schema", - } for key, value in content.items(): if key in top_level_keywords: continue @@ -371,7 +390,9 @@ def _content_is_abstract(content: dict[str, Any]) -> bool: def _content_is_final(content: dict[str, Any]) -> bool: return content.get("x-gts-final") is True - def _validate_schema_chain(self, gts_id: str) -> None: + def _validate_schema_chain( + self, gts_id: str, transient_schema: dict[str, Any] | None = None + ) -> None: """Validate OP#12: schema derivation chain compatibility.""" gid = GtsID(gts_id) segments = gid.gts_id_segments @@ -394,6 +415,13 @@ def _validate_schema_chain(self, gts_id: str) -> None: base_entity = self.get(base_id) derived_entity = self.get(derived_id) + derived_content = ( + transient_schema + if transient_schema is not None and derived_id == gts_id + else derived_entity.content + if derived_entity + else None + ) # Check x-gts-final: if the base type is final, derivation is not allowed. if ( @@ -413,14 +441,14 @@ def _validate_schema_chain(self, gts_id: str) -> None: raise ValueError( f"Base schema '{base_id}' not found for chain validation" ) - if not derived_entity or not isinstance(derived_entity.content, dict): - raise ValueError( + if not isinstance(derived_content, dict): + raise TypeError( f"Derived schema '{derived_id}' not found for chain validation" ) # Resolve both schemas (inline $refs) base_resolved = self._resolve_schema_refs(base_entity.content) - derived_resolved = self._resolve_schema_refs(derived_entity.content) + derived_resolved = self._resolve_schema_refs(derived_content) # Validate derivation compatibility (OP#12): accepted-instance-set # inclusion on declared schemas plus GTS admission rules. @@ -504,7 +532,9 @@ def _inline_refs( ] return node - def _build_effective_traits(self, gts_id: str) -> traits.EffectiveTraits: + def _build_effective_traits( + self, gts_id: str, transient_schema: dict[str, Any] | None = None + ) -> traits.EffectiveTraits: """Build OP#13 EffectiveTraits by walking the type's chain (root -> leaf).""" gid = GtsID(gts_id) segments = gid.gts_id_segments @@ -520,9 +550,15 @@ def _build_effective_traits(self, gts_id: str) -> traits.EffectiveTraits: for schema_id in chain_ids: entity = self.get(schema_id) - if not entity or not isinstance(entity.content, dict): + content = ( + transient_schema + if transient_schema is not None and schema_id == gts_id + else entity.content + if entity + else None + ) + if not isinstance(content, dict): continue - content = entity.content level_schemas: list[Any] = [] traits.collect_trait_schema_from_value(content, level_schemas) @@ -537,18 +573,32 @@ def _build_effective_traits(self, gts_id: str) -> traits.EffectiveTraits: traits.merge_rfc7396_into(merged_traits, level_traits) leaf = self.get(chain_ids[-1]) if chain_ids else None + leaf_content = ( + transient_schema + if transient_schema is not None and chain_ids[-1] == gts_id + else leaf.content + if leaf + else None + ) dialect = None - if leaf and isinstance(leaf.content, dict): - ds = leaf.content.get("$schema") + if isinstance(leaf_content, dict): + ds = leaf_content.get("$schema") if isinstance(ds, str): dialect = ds return traits.build_effective_traits(trait_schemas, merged_traits, dialect) - def _validate_traits(self, gts_id: str, is_abstract: bool) -> None: + def _validate_traits( + self, + gts_id: str, + is_abstract: bool, + transient_schema: dict[str, Any] | None = None, + ) -> None: """Validate OP#13: schema traits for a type.""" - effective = self._build_effective_traits(gts_id) - errors = effective.validate(check_unresolved=not is_abstract) + effective = self._build_effective_traits(gts_id, transient_schema) + errors = effective.validate( + check_unresolved=not is_abstract, reference_store=self + ) if errors: raise ValueError( f"Schema '{gts_id}' trait validation failed: " + "; ".join(errors) @@ -598,34 +648,14 @@ def validate_schema_basic(self, gts_id: str) -> None: # 3. Validate GTS keywords (x-gts-final, x-gts-abstract, placement) self._validate_gts_keywords(schema_content) - def validate_schema(self, gts_id: str) -> None: - """ - Full schema validation including: - 1. JSON Schema meta-schema validation - 2. x-gts-ref field validation - 3. GTS keyword validation (x-gts-final, x-gts-abstract, placement) - 4. Schema chain derivation validation (OP#12) - - Args: - gts_id: The GTS ID of the schema to validate - """ - if not gts_id.endswith("~"): + def validate_schema_content( + self, gts_id: str, schema_content: dict[str, Any] + ) -> None: + """Validate a schema using the registry only for its dependencies.""" + schema_id = GtsID(gts_id) + if not schema_id.is_type: raise ValueError(f"ID '{gts_id}' is not a schema (must end with '~')") - schema_entity = self.get(gts_id) - if not schema_entity: - raise StoreGtsSchemaNotFound(gts_id) - - if not schema_entity.is_schema: - raise ValueError(f"Entity '{gts_id}' is not a schema") - - schema_content = schema_entity.content - if not isinstance(schema_content, dict): - raise ValueError( # noqa: TRY004 - keep ValueError for API compatibility - f"Schema '{gts_id}' content must be a dictionary" - ) - - # Issue #25: strict check, no GTS IDs in $schema meta_schema_url = schema_content.get("$schema") if ( meta_schema_url @@ -636,21 +666,12 @@ def validate_schema(self, gts_id: str) -> None: f"Invalid $schema URL '{meta_schema_url}': must be a standard JSON Schema URL, not a GTS ID" ) - logger.info(f"Validating schema {gts_id}") - - # 1. Validate $ref fields - must be local (#...) or gts:// URIs + logger.info(f"Validating schema {schema_id.id}") self._validate_schema_refs(schema_content, "") - - # 2. Validate x-gts-ref fields - self._validate_schema_x_gts_refs(gts_id) - - # 3. Validate GTS keywords (x-gts-final, x-gts-abstract, placement) + self._validate_schema_x_gts_refs_content(schema_id.id, schema_content) self._validate_gts_keywords(schema_content) + self._validate_schema_chain(schema_id.id, schema_content) - # 4. Validate schema derivation chain (OP#12) - self._validate_schema_chain(gts_id) - - # 5. Validate against JSON Schema meta-schema try: from jsonschema import Draft7Validator from jsonschema.validators import validator_for @@ -661,15 +682,75 @@ def validate_schema(self, gts_id: str) -> None: else: Draft7Validator.check_schema(schema_content) - logger.info(f"Schema {gts_id} passed JSON Schema meta-schema validation") - except Exception as e: + logger.info( + f"Schema {schema_id.id} passed JSON Schema meta-schema validation" + ) + except Exception as error: raise ValueError( - f"JSON Schema validation failed for '{gts_id}': {e!s}" - ) from e + f"JSON Schema validation failed for '{schema_id.id}': {error!s}" + ) from error + + self._validate_traits( + schema_id.id, + self._content_is_abstract(schema_content), + schema_content, + ) - # 6. Validate traits (OP#13) - is_abstract = self._content_is_abstract(schema_content) - self._validate_traits(gts_id, is_abstract) + def validate_schema(self, gts_id: str) -> None: + """Validate a registered schema and all of its dependencies.""" + try: + schema_id = GtsID(gts_id) + except ValueError as error: + raise ValueError( + f"ID '{gts_id}' is not a schema (must end with '~')" + ) from error + if not schema_id.is_type: + raise ValueError(f"ID '{gts_id}' is not a schema (must end with '~')") + + schema_entity = self.get(schema_id.id) + if not schema_entity: + raise StoreGtsSchemaNotFound(schema_id.id) + if not schema_entity.is_schema: + raise ValueError(f"Entity '{schema_id.id}' is not a schema") + if not isinstance(schema_entity.content, dict): + raise ValueError( # noqa: TRY004 - keep ValueError for API compatibility + f"Schema '{schema_id.id}' content must be a dictionary" + ) + self.validate_schema_content(schema_id.id, schema_entity.content) + + def validate_instance_content(self, content: dict[str, Any], type_id: str) -> None: + """Validate unregistered instance content against a registered type schema.""" + schema_type = GtsID(type_id) + if not schema_type.is_type: + raise ValueError(f"ID '{type_id}' is not a schema (must end with '~')") + try: + schema = self.get_schema_content(schema_type.id) + except KeyError as error: + raise StoreGtsSchemaNotFound(schema_type.id) from error + + if isinstance(schema, dict) and self._content_is_abstract(schema): + raise ValueError( + f"type '{schema_type.id}' is abstract and cannot have direct instances" + ) + + schema_for_validation = _without_x_gts_ref(schema) + validator_class = validator_for(schema_for_validation) + validator = validator_class( + schema_for_validation, registry=self._create_reference_registry() + ) + validator.validate(content) + + x_gts_ref_validator = XGtsRefValidator(store=self) + x_gts_ref_errors = x_gts_ref_validator.validate_instance( + content, self._resolve_schema_refs(schema) + ) + if x_gts_ref_errors: + error_messages = [ + f"{err.field_path}: {err.reason}" for err in x_gts_ref_errors + ] + raise ValueError( + f"x-gts-ref validation failed: {'; '.join(error_messages)}" + ) def validate_instance( self, @@ -701,38 +782,11 @@ def validate_instance( raise StoreGtsObjectNotFound(gts_id) if not obj.type_id: raise StoreGtsSchemaForInstanceNotFound(lookup_id) - try: - schema = self.get_schema_content(obj.type_id) - except KeyError as e: - raise StoreGtsSchemaNotFound(obj.type_id) from e + if not isinstance(obj.content, dict): + raise TypeError(f"Instance '{lookup_id}' content must be a dictionary") logger.info(f"Validating instance {gts_id} against schema {obj.type_id}") - - # Check if the schema is abstract - abstract types cannot have direct instances - if isinstance(schema, dict) and self._content_is_abstract(schema): - raise ValueError( - f"type '{obj.type_id}' is abstract and cannot have direct instances" - ) - - schema_for_validation = _without_x_gts_ref(schema) - validator_class = validator_for(schema_for_validation) - validator = validator_class( - schema_for_validation, registry=self._create_reference_registry() - ) - validator.validate(obj.content) - - # Validate x-gts-ref constraints against the ref-resolved schema. - x_gts_ref_validator = XGtsRefValidator(store=self) - x_gts_ref_errors = x_gts_ref_validator.validate_instance( - obj.content, self._resolve_schema_refs(schema) - ) - if x_gts_ref_errors: - error_messages = [ - f"{err.field_path}: {err.reason}" for err in x_gts_ref_errors - ] - raise ValueError( - f"x-gts-ref validation failed: {'; '.join(error_messages)}" - ) + self.validate_instance_content(obj.content, obj.type_id) def cast( self, diff --git a/gts/src/gts/traits.py b/gts/src/gts/traits.py index bf4367d..2488680 100644 --- a/gts/src/gts/traits.py +++ b/gts/src/gts/traits.py @@ -19,6 +19,7 @@ import copy from typing import Any +from jsonschema import Draft7Validator, FormatChecker from jsonschema.validators import validator_for from . import derivation @@ -28,6 +29,8 @@ X_GTS_TRAITS = "x-gts-traits" MAX_RECURSION_DEPTH = 64 _MISSING = object() +_FORMAT_CHECKER = FormatChecker() +_FORMAT_CHECKER.checkers.update(Draft7Validator.FORMAT_CHECKER.checkers) class EffectiveTraits: @@ -51,7 +54,9 @@ def _has_schema(self) -> bool: def _has_explicit_values(self) -> bool: return isinstance(self.merged_traits, dict) and len(self.merged_traits) > 0 - def validate(self, check_unresolved: bool) -> list[str]: + def validate( + self, check_unresolved: bool, reference_store: Any | None = None + ) -> list[str]: """Return a list of error strings (empty means valid).""" errors = _validate_trait_schema_integrity(self.resolved_trait_schemas) if errors: @@ -76,7 +81,9 @@ def validate(self, check_unresolved: bool) -> list[str]: ] return [] - return _validate_trait_values(self.schema, self.values, check_unresolved) + return _validate_trait_values( + self.schema, self.values, check_unresolved, reference_store + ) # --- collection ------------------------------------------------------------ @@ -358,7 +365,7 @@ def _validate_traits_against_schema( try: cls = validator_for(validation_schema) - validator = cls(validation_schema) + validator = cls(validation_schema, format_checker=_FORMAT_CHECKER) for error in validator.iter_errors(effective_traits): errors.append(f"trait validation: {error.message}") except Exception as e: # noqa: BLE001 - surfaced as validation error message @@ -391,12 +398,15 @@ def _validate_traits_against_schema( def _validate_trait_values( - effective_traits_schema: Any, effective_traits: Any, check_unresolved: bool + effective_traits_schema: Any, + effective_traits: Any, + check_unresolved: bool, + reference_store: Any | None, ) -> list[str]: errors = _validate_traits_against_schema( effective_traits_schema, effective_traits, check_unresolved ) - xref = XGtsRefValidator() + xref = XGtsRefValidator(store=reference_store, require_registered_target=True) for err in xref.validate_instance(effective_traits, effective_traits_schema, ""): errors.append(f"trait x-gts-ref: {err.reason}") return errors diff --git a/gts/src/gts/x_gts_ref.py b/gts/src/gts/x_gts_ref.py index ddfc0ac..726258b 100644 --- a/gts/src/gts/x_gts_ref.py +++ b/gts/src/gts/x_gts_ref.py @@ -74,7 +74,9 @@ def __init__(self, field_path: str, value: Any, ref_pattern: str, reason: str): class XGtsRefValidator: """Validator for x-gts-ref constraints in GTS schemas.""" - def __init__(self, store: Any | None = None): + def __init__( + self, store: Any | None = None, require_registered_target: bool = False + ): """ Initialize validator. @@ -82,6 +84,7 @@ def __init__(self, store: Any | None = None): store: Optional GtsStore for resolving entity references """ self.store = store + self.require_registered_target = require_registered_target def validate_instance( self, instance: dict[str, Any], schema: dict[str, Any], instance_path: str = "" @@ -419,7 +422,9 @@ def _validate_gts_pattern( ) # Optionally check if entity exists in store - if self.store: + if self.store and ( + not self.require_registered_target or self.store.get(pattern) + ): entity = self.store.get(value) if not entity: return XGtsRefValidationError( diff --git a/tests/test_json_validation.py b/tests/test_json_validation.py new file mode 100644 index 0000000..8ada971 --- /dev/null +++ b/tests/test_json_validation.py @@ -0,0 +1,241 @@ +import json + +import pytest + +from gts._cli import main +from gts.entities import DEFAULT_GTS_CONFIG +from gts._json_validation import GtsJsonValidator + + +def test_validate_json_reports_all_document_errors(tmp_path): + (tmp_path / "broken.json").write_text('{ "id": "gts.broken', encoding="utf-8") + (tmp_path / "invalid-schema.json").write_text( + json.dumps( + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "gts://gts.example.catalog._.item.v1~", + "type": 3, + } + ), + encoding="utf-8", + ) + (tmp_path / "instance.json").write_text( + json.dumps( + { + "id": "gts.example.catalog._.item.v1~example.catalog._.one.v1", + "type": "gts.example.catalog._.item.v1~", + } + ), + encoding="utf-8", + ) + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert result.files == 3 + assert result.documents == 2 + assert result.schemas == 1 + assert result.instances == 1 + assert not result.ok + assert {issue.stage for issue in result.issues} >= { + "json", + "instance", + } + + +def test_validate_json_registers_type_only_instances(tmp_path): + type_id = "gts.example.catalog._.item.v1~" + (tmp_path / "schema.json").write_text( + json.dumps( + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": f"gts://{type_id}", + "type": "object", + "required": ["type", "name"], + "properties": {"type": {"const": type_id}, "name": {"type": "string"}}, + } + ), + encoding="utf-8", + ) + (tmp_path / "instance.json").write_text( + json.dumps({"type": type_id, "name": "anonymous"}), encoding="utf-8" + ) + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert result.ok + assert result.schemas == 1 + assert result.instances == 1 + + +def test_validate_json_cli_outputs_json_only(tmp_path, capsys): + input_path = tmp_path / "broken.json" + input_path.write_text('{ "id": "gts.broken', encoding="utf-8") + + with pytest.raises(SystemExit) as exc_info: + main(["validate-all", "--path", str(input_path)]) + assert exc_info.value.code == 1 + + captured = capsys.readouterr() + output = json.loads(captured.out) + assert output["ok"] is False + assert output["issues"][0]["file"] == str(input_path) + assert captured.err == "" + + +def test_malformed_schema_id_is_reported(tmp_path): + malformed = json.dumps({ + "$id": "gts://gtx.cli.core.test.bad.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + }) + (tmp_path / "bad.schema.json").write_text(malformed, encoding="utf-8") + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert not result.ok, "malformed GTS id should fail" + assert result.gts_entities == 0 + assert any( + i.stage == "registry" and "malformed" in i.message for i in result.issues + ), f"expected a malformed-id diagnostic, got: {result.issues}" + + +def test_incidental_prefix_mention_is_not_registered(tmp_path): + doc = json.dumps({"description": "see gts.foo.bar for details", "value": 42}) + (tmp_path / "unrelated.json").write_text(doc, encoding="utf-8") + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert result.ok, f"incidental mention should not fail: {result.issues}" + assert result.documents == 1 + assert result.gts_entities == 0 + assert not result.issues + + +def test_duplicate_entity_is_reported(tmp_path): + schema = json.dumps({ + "$id": "gts://gts.cli.core.test.base.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }) + (tmp_path / "a.schema.json").write_text(schema, encoding="utf-8") + (tmp_path / "b.schema.json").write_text(schema, encoding="utf-8") + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert not result.ok, "duplicate ids should fail" + assert any( + "Duplicate" in i.message for i in result.issues + ), f"expected a duplicate diagnostic, got: {result.issues}" + + +def test_non_gts_files_are_ignored(tmp_path): + schema = json.dumps({ + "$id": "gts://gts.cli.core.test.base.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }) + (tmp_path / "base.schema.json").write_text(schema, encoding="utf-8") + (tmp_path / "package.json").write_text( + '{"name": "pkg", "version": "1.0.0"}', encoding="utf-8" + ) + nm = tmp_path / "node_modules" + nm.mkdir() + (nm / "broken.json").write_text("{ this is not json ", encoding="utf-8") + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + assert result.ok, f"non-GTS files must be ignored: {result.issues}" + assert result.files == 1, "only the GTS schema should be processed" + assert result.schemas == 1 + + +def test_marker_heuristic_matches_expected_combinations(): + assert GtsJsonValidator._is_gts_marker('{"id": "gts.x.y.z.t.v1~a.b.c.d.v1.0"}') + assert GtsJsonValidator._is_gts_marker('{"$id": "gts://gts.x.y.z.t.v1~"}') + assert GtsJsonValidator._is_gts_marker( + '{"properties": {"p": {"x-gts-ref": "..."}}}' + ) + assert not GtsJsonValidator._is_gts_marker( + '{"name": "widgets", "version": "1.0.0"}' + ) + + +def test_schema_errors_ordered_by_depth_then_gts_id(tmp_path): + base_schema = json.dumps({ + "$id": "gts://gts.cli.core.test.base.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }) + (tmp_path / "base.schema.json").write_text(base_schema, encoding="utf-8") + + def invalid_base(seg): + return json.dumps({ + "$id": f"gts://gts.cli.core.test.{seg}.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "invalid_type", + }) + + invalid_leaf = json.dumps({ + "$id": "gts://gts.cli.core.test.base.v1~cli.core.test.leaf.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "invalid_type", + }) + + (tmp_path / "0_mmm.schema.json").write_text(invalid_base("mmm"), encoding="utf-8") + (tmp_path / "a_leaf.schema.json").write_text(invalid_leaf, encoding="utf-8") + (tmp_path / "z_aaa.schema.json").write_text(invalid_base("aaa"), encoding="utf-8") + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + schema_issues = [ + i for i in result.issues if i.stage in ("base-type", "derived-type") + ] + + assert len(schema_issues) == 3, f"issues: {result.issues}" + assert schema_issues[0].stage == "base-type" + assert schema_issues[0].file.endswith("z_aaa.schema.json") + assert schema_issues[1].stage == "base-type" + assert schema_issues[1].file.endswith("0_mmm.schema.json") + assert schema_issues[2].stage == "derived-type" + assert schema_issues[2].file.endswith("a_leaf.schema.json") + + +def test_instance_errors_ordered_by_gts_id(tmp_path): + base_schema = json.dumps({ + "$id": "gts://gts.cli.core.test.base.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }) + (tmp_path / "base.schema.json").write_text(base_schema, encoding="utf-8") + + def invalid_instance(seg): + return json.dumps({ + "id": f"gts.cli.core.test.base.v1~cli.app._.{seg}.v1.0" + }) + + (tmp_path / "z_alpha.json").write_text( + invalid_instance("alpha"), encoding="utf-8" + ) + (tmp_path / "a_zeta.json").write_text( + invalid_instance("zeta"), encoding="utf-8" + ) + + result = GtsJsonValidator(str(tmp_path), DEFAULT_GTS_CONFIG).validate() + + instance_issues = [i for i in result.issues if i.stage == "instance"] + + assert len(instance_issues) == 2, f"issues: {result.issues}" + assert instance_issues[0].file.endswith("z_alpha.json"), ( + f"alpha should be first: {instance_issues}" + ) + assert instance_issues[1].file.endswith("a_zeta.json"), ( + f"zeta should be second: {instance_issues}" + ) diff --git a/tests/test_ops.py b/tests/test_ops.py index 95ca333..0d30bbc 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -7,7 +7,7 @@ SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "gts.x.test._.foo.v1~", + "$id": "gts://gts.x.test._.foo.v1~", "type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"], @@ -56,13 +56,16 @@ def test_add_schema_missing_gts_id(self, ops): assert result.ok is False assert "Unable to detect GTS ID" in result.error - def test_add_schema_plain_gts_prefix_rejected_when_validate(self, ops): + def test_add_schema_plain_gts_prefix_rejected(self, ops): + # A schema $id MUST use the gts:// URI form. A bare gts. prefix yields + # no gts_id at the core entity layer, so registration is rejected + # regardless of the validate flag. schema = dict(SCHEMA) schema["$id"] = "gts.x.test._.foo.v1~" - result = ops.add_entity(schema, validate=True) - # $id doesn't start with gts:// -> rejected only if raw $id startswith "gts." - assert result.ok is False - assert "gts:// URI format" in result.error + for validate in (False, True): + result = ops.add_entity(schema, validate=validate) + assert result.ok is False + assert "Unable to detect GTS ID in schema" in result.error def test_add_instance_without_id_field_rejected(self, ops): result = ops.add_entity({"name": "hi"}) @@ -88,7 +91,7 @@ def test_add_instance_validate_failure_restores_previous(self, ops): def test_add_schema_validate_basic_failure(self, ops): bad_schema = { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "gts.x.test._.foo.v1~", + "$id": "gts://gts.x.test._.foo.v1~", "type": "object", "x-gts-ref": "notgts.*", } @@ -167,9 +170,7 @@ def test_match_true(self, ops): assert result.match is True def test_match_false(self, ops): - result = ops.match_id_pattern( - "gts.x.other._.foo.v1~", "gts.x.test.*" - ) + result = ops.match_id_pattern("gts.x.other._.foo.v1~", "gts.x.test.*") assert result.match is False def test_match_malformed_wildcard_candidate(self, ops): @@ -236,6 +237,73 @@ def test_validate_entity_invalid_id(self, ops): assert result.entity_type == "" +class TestValidateJson: + def test_validates_schema_without_storing_it(self, ops): + result = ops.validate_json(SCHEMA) + + assert result.ok is True + assert result.is_type_schema is True + assert ops.store.get(SCHEMA["$id"].removeprefix("gts://")) is None + + def test_validates_derived_schema_against_registered_parent(self, ops): + ops.add_entity(SCHEMA) + derived_schema = { + "$id": "gts://gts.x.test._.foo.v1~x.test._.bar.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "allOf": [{"$ref": "gts://gts.x.test._.foo.v1~"}], + } + + result = ops.validate_json(derived_schema) + + assert result.ok is True + assert ops.store.get("gts.x.test._.foo.v1~x.test._.bar.v1~") is None + + def test_validates_instance_against_explicit_type_without_storing_it(self, ops): + ops.add_entity(SCHEMA) + instance = {"name": "hi"} + + result = ops.validate_json(instance, explicit_type_id="gts.x.test._.foo.v1~") + + assert result.ok is True + assert result.type_id == "gts.x.test._.foo.v1~" + + def test_does_not_mutate_registry(self, ops, monkeypatch): + monkeypatch.setattr( + ops.store, + "register", + lambda entity: pytest.fail( + "transient validation must not register entities" + ), + ) + monkeypatch.setattr( + ops.store, + "unregister", + lambda entity_id: pytest.fail( + "transient validation must not unregister entities" + ), + ) + + result = ops.validate_json(SCHEMA) + + assert result.ok is True + + def test_rejects_missing_automatic_instance_type(self, ops): + result = ops.validate_json({"id": "gts.x.test._.missing_type.v1"}) + + assert result.ok is False + assert result.error == "Unable to determine instance type" + + def test_rejects_explicit_type_that_conflicts_with_body(self, ops): + result = ops.validate_json( + {"type": "gts.x.test._.other.v1~"}, + explicit_type_id="gts.x.test._.foo.v1~", + ) + + assert result.ok is False + assert "does not match path type" in result.error + + class TestSchemaGraphCompatibilityCast: def test_schema_graph(self, ops): ops.add_entity(SCHEMA) @@ -255,7 +323,9 @@ def test_cast_success(self, ops): assert result.error == "" def test_cast_error_wrapped(self, ops): - result = ops.cast("gts.x.test._.foo.v1~x.test._.missing.v1", "gts.x.test._.foo.v1~") + result = ops.cast( + "gts.x.test._.foo.v1~x.test._.missing.v1", "gts.x.test._.foo.v1~" + ) assert result.error != "" diff --git a/tests/test_schema_cast.py b/tests/test_schema_cast.py index 09aa0c4..454cf08 100644 --- a/tests/test_schema_cast.py +++ b/tests/test_schema_cast.py @@ -342,6 +342,20 @@ def test_forward_removed_enum_values_flagged(self): assert not ok assert any("removed enum values" in e for e in errors) + def test_backward_added_enum_constraint_flagged(self): + old = {"properties": {"a": {"allOf": [{}]}}} + new = {"properties": {"a": {"allOf": [{"enum": ["x"]}]}}} + ok, errors = GtsEntityCastResult._check_backward_compatibility(old, new) + assert not ok + assert any("added enum constraint" in e for e in errors) + + def test_forward_removed_enum_constraint_flagged(self): + old = {"properties": {"a": {"allOf": [{"enum": ["x"]}]}}} + new = {"properties": {"a": {"allOf": [{}]}}} + ok, errors = GtsEntityCastResult._check_forward_compatibility(old, new) + assert not ok + assert any("removed enum constraint" in e for e in errors) + def test_nested_object_errors_prefixed(self): old = {"properties": {"a": {"type": "object", "properties": {"b": {"type": "string"}}}}} new = {"properties": {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}} diff --git a/tests/test_server.py b/tests/test_server.py index da5e1c0..741a144 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -5,6 +5,7 @@ import asyncio import pytest +from fastapi.responses import JSONResponse from gts.ops import GtsOps from gts._server import GtsHttpServer, ValidateEntityRequest, _RequestLoggingMiddleware @@ -12,7 +13,7 @@ SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "gts.x.test._.foo.v1~", + "$id": "gts://gts.x.test._.foo.v1~", "type": "object", "properties": {"name": {"type": "string"}}, } @@ -190,7 +191,7 @@ def test_dispatch_skips_when_not_verbose(self, server): middleware = _RequestLoggingMiddleware(server.app, verbose=0) async def call_next(request): - return "response-sentinel" + return JSONResponse({"ok": True}) result = run(middleware.dispatch(request=None, call_next=call_next)) - assert result == "response-sentinel" + assert result.headers["connection"] == "close" diff --git a/tests/test_store.py b/tests/test_store.py index 3b03d29..778ccdc 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -245,7 +245,7 @@ def _create_store_with_schema_and_instance(self): schema = GtsEntity( content={ "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "gts.vendor.package.namespace.type.v1~", + "$id": "gts://gts.vendor.package.namespace.type.v1~", "type": "object", "properties": { "name": {"type": "string"}, @@ -316,7 +316,7 @@ def test_build_graph_simple(self): """Test building a simple graph.""" schema = GtsEntity( content={ - "$id": "gts.vendor.package.namespace.type.v1~", + "$id": "gts://gts.vendor.package.namespace.type.v1~", "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", }, diff --git a/tests/test_traits.py b/tests/test_traits.py index 576389f..72a8342 100644 --- a/tests/test_traits.py +++ b/tests/test_traits.py @@ -157,6 +157,26 @@ def test_valid_trait_values_pass(self): effective = build_effective_traits([schema], {"a": "hi"}, None) assert effective.validate(check_unresolved=True) == [] + def test_standard_trait_formats_are_enforced(self): + schema = { + "type": "object", + "properties": { + "email": {"type": "string", "format": "email"}, + "time": {"type": "string", "format": "time"}, + }, + } + + assert ( + build_effective_traits( + [schema], {"email": "user@example.com", "time": "10:30:00Z"}, None + ).validate(check_unresolved=True) + == [] + ) + errors = build_effective_traits( + [schema], {"email": "not-an-email", "time": "10:30:00Z"}, None + ).validate(check_unresolved=True) + assert any("is not a 'email'" in error for error in errors) + def test_invalid_trait_type_fails(self): schema = { "type": "object",