From a39e15b2a0cb3569aa61421f33c9d692ccb51abb Mon Sep 17 00:00:00 2001
From: Ilya Beda <ir4y.ix@gmail.com>
Date: Thu, 29 Oct 2026 19:11:41 +1100
Subject: [PATCH 1/2] WIP more featires

support json source format
support fixture autocreate
support profile validation
---
 README.md                        |   2 +-
 testscript_eval/assertation.py   |  80 +++++++++++++++++++++---
 testscript_eval/eval.py          | 103 +++++++++++++++++++++++++------
 testscript_eval/operations.py    |   6 ++
 testscript_eval/pytest_plugin.py |   6 +-
 5 files changed, 166 insertions(+), 31 deletions(-)

diff --git a/README.md b/README.md
index 56aeb0c..fd41cc5 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,7 @@ docker run -v "$(pwd)/resources:/app" --env-file ./env/testscript bedasoftware/t
 ```
 ### Restrictions
 1. TestScript files should be stored in the TestScript folder;
-2. TestScript files should be in YAML format.
+2. TestScript files should be in YAML (`.yaml`) or JSON (`.json`) format.
 ## Recommendations
 Highly recommend using testscript-eval-py alongside [Kaitenzushi](https://github.com/beda-software/kaitenzushi/) to enhance the TestScript development experience.
 ## References
diff --git a/testscript_eval/assertation.py b/testscript_eval/assertation.py
index 33eb37c..416d0f6 100644
--- a/testscript_eval/assertation.py
+++ b/testscript_eval/assertation.py
@@ -1,3 +1,4 @@
+import json
 import logging
 import re
 
@@ -61,8 +62,52 @@ def response(_assert_name, assertation, result, resource, var):
         raise Exception(f"Response code {response.status} is not supported")
 
 
-def validate_profile_id(_assert_name, assertation, result, resource, var):
-    logging.warning("Profile validation is not implemented yet")
+def resolve_profile(profile_id, profiles):
+    # validateProfileId is the id of a TestScript.profile entry; a canonical URL given
+    # directly instead (as fhir-frog TestScripts do) is used as is
+    for profile in profiles:
+        if profile.get("id") == profile_id:
+            return profile["reference"]
+    if "/" in profile_id:
+        return profile_id
+    raise Exception(f"validateProfileId '{profile_id}' does not match any TestScript.profile id")
+
+
+async def validate_profile_id(assertation, resource, client, profiles):
+    profile_url = resolve_profile(assertation["validateProfileId"], profiles)
+    assert isinstance(resource, dict) and "resourceType" in resource, (
+        f"No resource to validate against {profile_url}"
+    )
+    # Pass the profile both ways: as the $validate `profile` parameter, and in meta.profile,
+    # which servers that ignore the parameter (e.g. Aidbox) validate against
+    meta = resource.get("meta", {})
+    payload = {
+        **resource,
+        "meta": {**meta, "profile": list(dict.fromkeys([*meta.get("profile", []), profile_url]))},
+    }
+    response, outcome = await client.execute(
+        f"{resource['resourceType']}/$validate",
+        method="post",
+        data=payload,
+        params={"profile": profile_url},
+    )
+    if not isinstance(outcome, dict):
+        # Non-2xx: the body comes back wrapped in an exception object
+        try:
+            outcome = json.loads(str(outcome))
+        except ValueError:
+            raise AssertionError(
+                f"$validate against {profile_url} failed with status {response.status}: {outcome}"
+            )
+    errors = [
+        f"[{issue.get('severity')}] {', '.join(issue.get('expression', []))}: "
+        f"{issue.get('diagnostics') or issue.get('details', {}).get('text', '')}"
+        for issue in outcome.get("issue", [])
+        if issue.get("severity") in ("error", "fatal")
+    ]
+    assert response.status < 400 and not errors, (
+        f"Validation against {profile_url} failed (status {response.status}):\n" + "\n".join(errors)
+    )
 
 
 def parse_variable_in_string(string: str, variables):
@@ -101,6 +146,22 @@ def compare_response_expression_to_value(_assert_name, assertation, result, reso
     operations.eval(operator, expression_result[0], value)
 
 
+def expression(_assert_name, assertation, result, resource, var):
+    # An expression with a value is compared by the "value" rule, and one with
+    # compareToSourceId by the "compareToSourceExpression" rule
+    if "value" in assertation or "compareToSourceId" in assertation:
+        return
+    # Without a value the expression is a boolean condition: operator "eval",
+    # which is also the default (https://hl7.org/fhir/R5/testing.html#assertion-table)
+    operator = assertation.get("operator", "eval")
+    if operator != "eval":
+        raise NotImplementedAssert(f"Operator {operator} without a value is not implemented")
+    fixtures = var["--fixtures--"]
+    id = assertation.get("sourceId")
+    source = fixtures[id] if id else resource
+    operations.eval(operator, fhirpath(source, assertation["expression"], {}), None)
+
+
 def minimum_id(_assert_name, assertation, result, resource, var):
     fixtures = var["--fixtures--"]
     id = assertation["minimumId"]
@@ -125,8 +186,8 @@ def navigation_links(_assert_name, assertation, result, resource, var):
     "responseCode": response_code,
     "resource": resource,
     "path": path,
+    "expression": expression,
     "response": response,
-    "validateProfileId": validate_profile_id,
     "compareToSourceExpression": compare_to_source_expression,
     "minimumId": minimum_id,
     "contentType": content_type,
@@ -139,15 +200,18 @@ def navigation_links(_assert_name, assertation, result, resource, var):
 }
 
 
-def eval(assertation, result, resource, var, fixtures):
+async def eval(assertation, result, resource, var, fixtures, client, profiles):
     # TODO rid of temporary hack
     # pass fixtures as separated context item
     var["--fixtures--"] = fixtures
     logging.warning("%s", assertation["description"])
+    if assertation.get("direction") == "request":
+        # Skip all request tests
+        logging.warning("Skip request validation")
+        return
     for assert_name, assert_eval in assert_rules.items():
-        if assertation.get("direction") == "request":
-            # Skip all request tests
-            logging.warning("Skip request validation")
-            continue
         if assert_name in assertation:
             assert_eval(assert_name, assertation, result, resource, var)
+    # Validation calls the server, so it is not one of the synchronous assert_rules
+    if "validateProfileId" in assertation:
+        await validate_profile_id(assertation, resource, client, profiles)
diff --git a/testscript_eval/eval.py b/testscript_eval/eval.py
index bb6546e..a424e3e 100644
--- a/testscript_eval/eval.py
+++ b/testscript_eval/eval.py
@@ -20,10 +20,6 @@
 async def setup_fixtures(client: RawResultAsyncFHIRClient, definition):
     fixtures = {}
     for f in definition.get("fixture", []):
-        if f.get("autocreate", False):
-            raise Exception("Autocreate is not supported")
-        if f.get("autodelete", False):
-            raise Exception("Autocreate is not supported")
         reference = f["resource"]["reference"]
         fixture_id = f["id"]
         if reference.startswith("#"):
@@ -49,6 +45,61 @@ async def setup_fixtures(client: RawResultAsyncFHIRClient, definition):
     return fixtures
 
 
+async def autocreate_fixtures(client: RawResultAsyncFHIRClient, definition, fixtures, created):
+    # `created` is filled one fixture at a time, so if a later create fails
+    # the fixtures already on the server are still autodeleted
+    for f in definition.get("fixture", []):
+        if not f.get("autocreate", False):
+            continue
+        fixture_id = f["id"]
+        fixture = fixtures[fixture_id]
+        logging.warning("Autocreate fixture %s", fixture_id)
+        # A fixture with an id is created under that id (update as create), so references
+        # between fixtures (e.g. Patient/<id>) resolve on the server as written
+        if "id" in fixture:
+            path, method = f"{fixture['resourceType']}/{fixture['id']}", "put"
+        else:
+            path, method = fixture["resourceType"], "post"
+        response, resource = await client.execute(path, method=method, data=fixture)
+        if not 200 <= response.status < 300:
+            raise Exception(
+                f"Autocreate of fixture '{fixture_id}' failed with status {response.status}: "
+                f"{resource}"
+            )
+        if not resource:
+            # No body returned (e.g. Prefer: return=minimal): take the id from Location
+            # ([base]/Type/id/_history/vid)
+            parts = urlsplit(response.headers["Location"]).path.strip("/").split("/")
+            if "_history" in parts:
+                parts = parts[: parts.index("_history")]
+            resource = {**fixture, "id": parts[-1]}
+        # Later references, variables and autodelete must use the server-assigned id
+        fixtures[fixture_id] = resource
+        created.append(fixture_id)
+
+
+async def autodelete_fixtures(client: RawResultAsyncFHIRClient, definition, fixtures, created):
+    for f in reversed(definition.get("fixture", [])):
+        if not f.get("autodelete", False):
+            continue
+        fixture_id = f["id"]
+        # An autocreate fixture whose create did not happen has nothing to delete
+        if f.get("autocreate", False) and fixture_id not in created:
+            continue
+        fixture = fixtures.get(fixture_id)
+        if not fixture or "id" not in fixture:
+            logging.warning("Autodelete fixture %s skipped: it has no id", fixture_id)
+            continue
+        logging.warning("Autodelete fixture %s", fixture_id)
+        response, _resource = await client.execute(
+            f"{fixture['resourceType']}/{fixture['id']}", method="delete"
+        )
+        if not (200 <= response.status < 300 or response.status in (404, 410)):
+            logging.warning(
+                "Autodelete of fixture %s failed with status %s", fixture_id, response.status
+            )
+
+
 def setup_variables(definition, fixtures, env):
     variables = {}
     for var in definition.get("variable", []):
@@ -67,17 +118,19 @@ async def setup(client, definition, fixtures, variables):
         return
 
     actions = definition["setup"]["action"]
-    return await eval_actions(client, actions, fixtures, variables)
+    return await eval_actions(client, actions, fixtures, variables, definition.get("profile", []))
 
 
 async def teardown(client, definition, fixtures, variables):
     if "teardown" not in definition:
         return
     actions = definition["teardown"]["action"]
-    return await eval_actions(client, actions, fixtures, variables)
+    return await eval_actions(client, actions, fixtures, variables, definition.get("profile", []))
 
 
-async def eval_actions(client: RawResultAsyncFHIRClient, actions, fixtures, variables):
+async def eval_actions(
+    client: RawResultAsyncFHIRClient, actions, fixtures, variables, profiles
+):
     response = None
     resource = None
     for action in actions:
@@ -129,7 +182,9 @@ async def eval_actions(client: RawResultAsyncFHIRClient, actions, fixtures, vari
                 fixtures[operation["responseId"]] = resource
         else:
             assert response
-            assertation.eval(action["assert"], response, resource, variables, fixtures)
+            await assertation.eval(
+                action["assert"], response, resource, variables, fixtures, client, profiles
+            )
 
 
 async def eval(definition, env):
@@ -141,18 +196,26 @@ async def eval(definition, env):
 
     logging.warning("Loading fixtures")
     fixtures = await setup_fixtures(client, definition)
-    logging.warning("Loading variables")
-    variables = setup_variables(definition, fixtures, env)
+    created = []
 
     try:
-        logging.warning("Setup")
-        await setup(client, definition, fixtures, variables)
-
-        logging.warning("Testing")
-        for test in definition.get("test", []):
-            logging.warning("Test %s, %s", test["name"], test["description"])
-            test_actions = test.get("action", [])
-            await eval_actions(client, test_actions, fixtures, variables)
+        await autocreate_fixtures(client, definition, fixtures, created)
+        logging.warning("Loading variables")
+        variables = setup_variables(definition, fixtures, env)
+
+        try:
+            logging.warning("Setup")
+            await setup(client, definition, fixtures, variables)
+
+            logging.warning("Testing")
+            for test in definition.get("test", []):
+                logging.warning("Test %s, %s", test["name"], test["description"])
+                test_actions = test.get("action", [])
+                await eval_actions(
+                    client, test_actions, fixtures, variables, definition.get("profile", [])
+                )
+        finally:
+            logging.warning("Teardown")
+            await teardown(client, definition, fixtures, variables)
     finally:
-        logging.warning("Teardown")
-        await teardown(client, definition, fixtures, variables)
+        await autodelete_fixtures(client, definition, fixtures, created)
diff --git a/testscript_eval/operations.py b/testscript_eval/operations.py
index 34a715a..8eeb8c6 100644
--- a/testscript_eval/operations.py
+++ b/testscript_eval/operations.py
@@ -33,7 +33,13 @@ def not_empty(_operation, source, _value):
     assert source is not None
 
 
+def evaluate(_operation, source, _value):
+    # `source` is the FHIRPath result collection of a boolean expression
+    assert source == [True], f"Expression evaluated to {source}, expected [True]"
+
+
 operations = {
+    "eval": evaluate,
     "exists": exists,
     "contains": contains,
     "in": contains,
diff --git a/testscript_eval/pytest_plugin.py b/testscript_eval/pytest_plugin.py
index 3a30fb7..3f233de 100644
--- a/testscript_eval/pytest_plugin.py
+++ b/testscript_eval/pytest_plugin.py
@@ -1,4 +1,5 @@
 import asyncio
+import json
 import os
 
 import pytest
@@ -9,13 +10,14 @@
 
 
 def pytest_collect_file(parent, file_path):
-    if file_path.suffix == ".yaml" and file_path.parent.name == "TestScript":
+    if file_path.suffix in (".yaml", ".json") and file_path.parent.name == "TestScript":
         return TestScriptFile.from_parent(parent, path=file_path)
 
 
 class TestScriptFile(pytest.File):
     def collect(self):
-        raw = yaml.safe_load(self.path.open())
+        with self.path.open() as f:
+            raw = json.load(f) if self.path.suffix == ".json" else yaml.safe_load(f)
         yield TestScriptItem.from_parent(self, name=raw["name"], spec=raw)
 
 

From b377b98f4cb4e853c36502ab8f97646a8ad1ddac Mon Sep 17 00:00:00 2001
From: Ilya Beda <ir4y.ix@gmail.com>
Date: Thu, 29 Oct 2026 19:23:10 +1100
Subject: [PATCH 2/2] WIP more features

Update fhirpath
support warningOnly
---
 Pipfile                        |  2 +-
 Pipfile.lock                   | 30 ++++++++-----
 pyproject.toml                 |  8 ++--
 testscript_eval/assertation.py | 81 +++++++++++++++++++++++-----------
 testscript_eval/eval.py        |  9 ++--
 testscript_eval/misc.py        |  8 ++++
 testscript_eval/operations.py  | 29 +++++++-----
 7 files changed, 111 insertions(+), 56 deletions(-)

diff --git a/Pipfile b/Pipfile
index 61d5cea..d3e8cfc 100644
--- a/Pipfile
+++ b/Pipfile
@@ -12,7 +12,6 @@ ipython = "~=8.11.0"
 
 [packages]
 fhirpy = "~=1.3.1"
-fhirpathpy = "~=0.1.2"
 pytest = "~=7.2.2"
 pytest-asyncio = "~=0.20.3"
 jsonpath-ng = "~=1.5.3"
@@ -20,6 +19,7 @@ funcy = "~=1.18"
 aiofiles = "~=23.1.0"
 pyyaml = "~=6.0"
 pyyaml-include = "~=1.3"
+fhirpathpy = "~=2.2.4"
 
 [requires]
 python_version = "3.11"
diff --git a/Pipfile.lock b/Pipfile.lock
index bdba3a0..7c73fa3 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "b141d20d0647c48132cb444bf8e624dc1f6b8b131cb0cfa619286a9f077d089e"
+            "sha256": "e0ae3ce6a58cee63c44dbdfd09f746997b5efbfddfb87643d9519c9b8c0f3d95"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -127,9 +127,10 @@
         },
         "antlr4-python3-runtime": {
             "hashes": [
-                "sha256:15793f5d0512a372b4e7d2284058ad32ce7dd27126b105fb0b2245130445db33"
+                "sha256:909b647e1d2fc2b70180ac586df3933e38919c85f98ccc656a96cd3f25ef3916",
+                "sha256:fe3835eb8d33daece0e799090eda89719dbccee7aa39ef94eed3818cafa5a7e8"
             ],
-            "version": "==4.8"
+            "version": "==4.13.2"
         },
         "async-timeout": {
             "hashes": [
@@ -246,11 +247,12 @@
         },
         "fhirpathpy": {
             "hashes": [
-                "sha256:0ade7a096cfc9398aff05dec8fb0369532a78ac8ebe7a9944bf9f0956ce817ad",
-                "sha256:47f9abd27e378ac0aa7ac465eadff56f1cfa74a53a16b7ec76a03b00ec71a6ef"
+                "sha256:5ea363963598a9c8c23425b6594986dc8c8e33600656b70639e088f0c62c355d",
+                "sha256:ff592994edb7a30fd499e1462f3f7ec6d83568df58487e9d522ead2f7a85ffdb"
             ],
             "index": "pypi",
-            "version": "==0.1.2"
+            "markers": "python_version >= '3.10'",
+            "version": "==2.2.4"
         },
         "fhirpy": {
             "hashes": [
@@ -492,6 +494,14 @@
             "index": "pypi",
             "version": "==0.20.3"
         },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
+                "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==2.9.0.post0"
+        },
         "pytz": {
             "hashes": [
                 "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0",
@@ -563,11 +573,11 @@
         },
         "six": {
             "hashes": [
-                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
-                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+                "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274",
+                "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.16.0"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==1.17.0"
         },
         "urllib3": {
             "hashes": [
diff --git a/pyproject.toml b/pyproject.toml
index 7bc0ad0..c55a47b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -9,7 +9,7 @@ readme = "README.md"
 license = { file = "LICENSE" }
 keywords = ["fhir"]
 authors = [{ name = "beda.software", email = "ilya@beda.software" }]
-dependencies = ["requests>=2.25.1", "aiohttp>=3.6.3", "pytz", "aiofiles", "fhirpathpy", "pyyaml-include", "jsonpath_ng", "funcy", "fhirpy", "pytest_asyncio"]
+dependencies = ["requests>=2.25.1", "aiohttp>=3.6.3", "pytz", "aiofiles", "fhirpathpy~=2.2.4", "pyyaml-include", "jsonpath_ng", "funcy", "fhirpy", "pytest_asyncio"]
 dynamic = ["version"]
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -17,13 +17,11 @@ classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 
 [tool.black]
 line-length = 100
@@ -50,7 +48,7 @@ known_first_party = ["testscript_eval"]
 
 [tool.pytest.ini_options]
 minversion = "7.0"
-addopts = "-ra -q --color=yes --disable-warnings"
+addopts = "-ra -q --color=yes"
 testpaths = ["tests"]
 log_cli = true
 log_cli_level = "INFO"
diff --git a/testscript_eval/assertation.py b/testscript_eval/assertation.py
index 416d0f6..5e92369 100644
--- a/testscript_eval/assertation.py
+++ b/testscript_eval/assertation.py
@@ -1,18 +1,23 @@
 import json
 import logging
 import re
+import warnings
 
-from fhirpathpy import evaluate as fhirpath
 from jsonpath_ng.ext import parse
 
 from . import operations
-from .misc import resolve_string_template
+from .misc import fhirpath, resolve_string_template
+from .operations import check
 
 
 class NotImplementedAssert(Exception):
     pass
 
 
+class TestScriptWarning(UserWarning):
+    """A failed warningOnly assert: reported, but does not fail the test script."""
+
+
 def not_implemented_assert(assert_name, _assertation, _result, _resource, _var):
     raise NotImplementedAssert(f"Assert rule {assert_name} is not implemented")
 
@@ -32,10 +37,11 @@ def response_code(_assert_name, assertation, result, _resource, _var):
 
 def resource(_assert_name, assertation, _result, resource, _var):
     resource_type = assertation["resource"]
-    if assertation.get("operator", "equals") == "equals":
-        assert resource_type == resource["resourceType"]
-    else:
-        assert resource_type == resource["resourceType"]
+    check(isinstance(resource, dict), "No resource in the response")
+    check(
+        resource_type == resource.get("resourceType"),
+        f"Expected resource {resource_type}, got {resource.get('resourceType')}",
+    )
 
 
 def path(_assert_name, assertation, result, resource, var):
@@ -51,15 +57,15 @@ def path(_assert_name, assertation, result, resource, var):
 def response(_assert_name, assertation, result, resource, var):
     response = assertation["response"]
     if response == "okay":
-        assert result.status == 200, f"Expected status 200 (okay), got {result.status}"
+        check(result.status == 200, f"Expected status 200 (okay), got {result.status}")
     elif response == "notFound":
-        assert result.status == 404, f"Expected status 404 (notFound), got {result.status}"
+        check(result.status == 404, f"Expected status 404 (notFound), got {result.status}")
     elif response == "bad":
-        assert result.status == 400, f"Expected status 400 (bad), got {result.status}"
+        check(result.status == 400, f"Expected status 400 (bad), got {result.status}")
     elif response == "created":
-        assert result.status == 201, f"Expected status 201 (created), got {result.status}"
+        check(result.status == 201, f"Expected status 201 (created), got {result.status}")
     else:
-        raise Exception(f"Response code {response.status} is not supported")
+        raise Exception(f"Response code {response} is not supported")
 
 
 def resolve_profile(profile_id, profiles):
@@ -75,8 +81,9 @@ def resolve_profile(profile_id, profiles):
 
 async def validate_profile_id(assertation, resource, client, profiles):
     profile_url = resolve_profile(assertation["validateProfileId"], profiles)
-    assert isinstance(resource, dict) and "resourceType" in resource, (
-        f"No resource to validate against {profile_url}"
+    check(
+        isinstance(resource, dict) and "resourceType" in resource,
+        f"No resource to validate against {profile_url}",
     )
     # Pass the profile both ways: as the $validate `profile` parameter, and in meta.profile,
     # which servers that ignore the parameter (e.g. Aidbox) validate against
@@ -91,22 +98,33 @@ async def validate_profile_id(assertation, resource, client, profiles):
         data=payload,
         params={"profile": profile_url},
     )
+    # No validation happened (auth, $validate unsupported, server error): an engine/infrastructure
+    # error, never a check failure, so warningOnly does not hide it
+    if response.status in (401, 403, 404, 405) or response.status >= 500:
+        raise Exception(
+            f"$validate against {profile_url} could not run: status {response.status}: {outcome}"
+        )
     if not isinstance(outcome, dict):
         # Non-2xx: the body comes back wrapped in an exception object
         try:
             outcome = json.loads(str(outcome))
         except ValueError:
-            raise AssertionError(
-                f"$validate against {profile_url} failed with status {response.status}: {outcome}"
-            )
+            outcome = None
+    if not isinstance(outcome, dict) or outcome.get("resourceType") != "OperationOutcome":
+        raise Exception(
+            f"$validate against {profile_url} returned no OperationOutcome "
+            f"(status {response.status}): {outcome}"
+        )
     errors = [
         f"[{issue.get('severity')}] {', '.join(issue.get('expression', []))}: "
         f"{issue.get('diagnostics') or issue.get('details', {}).get('text', '')}"
         for issue in outcome.get("issue", [])
         if issue.get("severity") in ("error", "fatal")
     ]
-    assert response.status < 400 and not errors, (
-        f"Validation against {profile_url} failed (status {response.status}):\n" + "\n".join(errors)
+    # A 2xx, or a 4xx such as 400/422 with an OperationOutcome, is the validation result
+    check(
+        response.status < 400 and not errors,
+        f"Validation against {profile_url} failed (status {response.status}):\n" + "\n".join(errors),
     )
 
 
@@ -137,6 +155,7 @@ def compare_response_expression_to_value(_assert_name, assertation, result, reso
     fixture = fixtures[id] if id else resource
     expression = assertation["expression"]
     expression_result = fhirpath(fixture, expression, {})
+    check(expression_result, f"Expression '{expression}' returned no value")
     if len(expression_result) > 1:
         logging.warning("Expression '%s' returns more then one value")
     operator = assertation.get("operator", "equals")
@@ -166,8 +185,9 @@ def minimum_id(_assert_name, assertation, result, resource, var):
     fixtures = var["--fixtures--"]
     id = assertation["minimumId"]
     fixture = fixtures[id]
+    check(isinstance(resource, dict), "No resource in the response")
     for k, v in fixture.items():
-        assert resource[k] == v
+        check(k in resource and resource[k] == v, f"minimumId {id}: {k} does not match")
 
 
 def content_type(_assert_name, assertation, result, resource, var):
@@ -178,7 +198,8 @@ def content_type(_assert_name, assertation, result, resource, var):
 
 
 def navigation_links(_assert_name, assertation, result, resource, var):
-    assert "link" in resource
+    check(isinstance(resource, dict), "No resource in the response")
+    check("link" in resource, "Response has no navigation links")
 
 
 assert_rules = {
@@ -209,9 +230,17 @@ async def eval(assertation, result, resource, var, fixtures, client, profiles):
         # Skip all request tests
         logging.warning("Skip request validation")
         return
-    for assert_name, assert_eval in assert_rules.items():
-        if assert_name in assertation:
-            assert_eval(assert_name, assertation, result, resource, var)
-    # Validation calls the server, so it is not one of the synchronous assert_rules
-    if "validateProfileId" in assertation:
-        await validate_profile_id(assertation, resource, client, profiles)
+    try:
+        for assert_name, assert_eval in assert_rules.items():
+            if assert_name in assertation:
+                assert_eval(assert_name, assertation, result, resource, var)
+        # Validation calls the server, so it is not one of the synchronous assert_rules
+        if "validateProfileId" in assertation:
+            await validate_profile_id(assertation, resource, client, profiles)
+    except AssertionError as e:
+        # A failed check of a warningOnly assert is reported and the script goes on; anything
+        # else (engine, script or infrastructure errors) is not an AssertionError and still raises
+        if not assertation.get("warningOnly", False):
+            raise
+        logging.warning("warningOnly assert failed: %s: %s", assertation["description"], e)
+        warnings.warn(TestScriptWarning(f"{assertation['description']}: {e}"))
diff --git a/testscript_eval/eval.py b/testscript_eval/eval.py
index a424e3e..7ee50c4 100644
--- a/testscript_eval/eval.py
+++ b/testscript_eval/eval.py
@@ -5,11 +5,10 @@
 
 import aiofiles
 import yaml
-from fhirpathpy import evaluate as fhirpath
 from yamlinclude import YamlIncludeConstructor
 
 from testscript_eval import assertation
-from testscript_eval.misc import RawResultAsyncFHIRClient, resolve_string_template
+from testscript_eval.misc import RawResultAsyncFHIRClient, fhirpath, resolve_string_template
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__name__))
 YamlIncludeConstructor.add_to_loader_class(
@@ -109,7 +108,8 @@ def setup_variables(definition, fixtures, env):
             variables[name] = fhirpath(context, var["expression"], {})[0]
         else:
             variables[name] = env.get(name, var.get("defaultValue"))
-            assert variables[name], f"Missing {name} variable"
+            if not variables[name]:
+                raise Exception(f"Missing {name} variable")
     return variables
 
 
@@ -181,7 +181,8 @@ async def eval_actions(
             if "responseId" in operation:
                 fixtures[operation["responseId"]] = resource
         else:
-            assert response
+            if response is None:
+                raise Exception("Assert has no preceding operation response")
             await assertation.eval(
                 action["assert"], response, resource, variables, fixtures, client, profiles
             )
diff --git a/testscript_eval/misc.py b/testscript_eval/misc.py
index b5e7788..318786a 100644
--- a/testscript_eval/misc.py
+++ b/testscript_eval/misc.py
@@ -1,6 +1,8 @@
 import json
 
 import aiohttp
+from fhirpathpy import evaluate
+from fhirpathpy.models import models
 from fhirpy import AsyncFHIRClient
 from fhirpy.base.exceptions import OperationOutcome, ResourceNotFound
 from fhirpy.base.utils import AttrDict
@@ -8,6 +10,12 @@
 from funcy.types import is_list, is_mapping
 
 
+def fhirpath(resource, expression, context):
+    # The FHIR R4 model lets FHIRPath resolve choice elements (e.g. `value` for
+    # valueCodeableConcept) and types (ofType, is, as) on plain JSON resources
+    return evaluate(resource, expression, context, models["r4"])
+
+
 def walk_dict(d):
     for k, v in d.items():
         if is_list(v):
diff --git a/testscript_eval/operations.py b/testscript_eval/operations.py
index 8eeb8c6..09cff84 100644
--- a/testscript_eval/operations.py
+++ b/testscript_eval/operations.py
@@ -5,6 +5,13 @@ class NotImplementedOperation(Exception):
     pass
 
 
+def check(condition, message):
+    # A failed check is the only thing raised as AssertionError, so warningOnly can tell it
+    # apart from engine errors; explicit raise, as `python -O` strips bare asserts
+    if not condition:
+        raise AssertionError(message)
+
+
 def not_implemented_operation(operation, source, value):
     raise NotImplementedOperation(f"Operation {operation} is not implemented")
 
@@ -13,29 +20,31 @@ def contains(_operation, source, values):
     match = False
     for value in values.split(","):
         match = match or value in source
-    assert match, f"{source} not in {values}"
+    check(match, f"{source} not in {values}")
 
 
 def exists(_operation, source, _value):
-    if is_list(source) and len(source) == 0:
-        raise Exception("Exist validation issue")
-    elif is_mapping(source) and source == {}:
-        raise Exception("Exist validation issue")
-    elif source is None:
-        raise Exception("Exist validation issue")
+    check(
+        not (
+            (is_list(source) and len(source) == 0)
+            or (is_mapping(source) and source == {})
+            or source is None
+        ),
+        "Exist validation issue",
+    )
 
 
 def equals(_operation, source, value):
-    assert source == value, f"Got {source}, expected {value}"
+    check(source == value, f"Got {source}, expected {value}")
 
 
 def not_empty(_operation, source, _value):
-    assert source is not None
+    check(source is not None, "Got no value, expected a non-empty one")
 
 
 def evaluate(_operation, source, _value):
     # `source` is the FHIRPath result collection of a boolean expression
-    assert source == [True], f"Expression evaluated to {source}, expected [True]"
+    check(source == [True], f"Expression evaluated to {source}, expected [True]")
 
 
 operations = {