diff --git a/providers/openfeature-provider-flagd/README.md b/providers/openfeature-provider-flagd/README.md index 8a49d1dc..5dc82f90 100644 --- a/providers/openfeature-provider-flagd/README.md +++ b/providers/openfeature-provider-flagd/README.md @@ -190,6 +190,46 @@ api.set_provider( ) ``` +## Provider conformance suite + +This provider runs the [OpenFeature Provider Conformance Suite][tck] against a flagd-testbed stack, +once per resolver, in `tests/tck`. The suite owns the container stack: `tests/tck/conftest.py` +declares a Compose file and the two ports the resolvers connect to, and nothing else. + +`tests/tck/docker-compose.yaml` is one definition of the backend for the whole repository, and the +OFREP adoption carries a byte-identical copy — each provider package publishes its own distribution +and must not read the other's files, so the two are kept in step by `diff` rather than by sharing a +path. Change one, copy it to the other. + +**It is excluded from the default build, and a maintainer runs it by hand before merging a change to +it.** + +``` +poe test-tck # both resolvers, needs Docker +poe test # everything else, which is what CI runs +``` + +The exclusion lives in `pyproject.toml`: `--ignore=tests/tck` on the two tasks `build.yml` reaches, +with the reason in a comment above them. Why a conformance suite is not a required gate is +[Appendix F, "Running the suite in CI"][appendix-f], and is not restated here. + +Two things that are this provider's rather than the policy's: + +- **Docker is not what decides it.** `tests/e2e` needs Docker too and does run in the default build. + What decides it is the run: **8 failed, 119 passed, 3 skipped**, being three canonical flags that + no released flagd-testbed seeds yet, failing on each resolver, plus two genuine provider-side + gaps — one per resolver, and only one of them carries a known-deviation entry. + `tests/tck/conftest.py` accounts for all eight individually, with the flag or requirement each one + turns on, so a reviewer running the suite can tell a new failure from a known one and the number + above is what to expect. +- **The default build still collects the suite** — `poe test` and `poe test-cov` end in + `pytest tests/tck --collect-only`, which imports every module and starts no container. An excluded + suite that has quietly stopped importing against the harness is worse than one that runs and + fails, and `mypy` here is configured over `src` alone, so nothing else would notice. + +[tck]: ../../tools/openfeature-tck/README.md +[appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md + ## License Apache 2.0 - See [LICENSE](./LICENSE) for more information. diff --git a/providers/openfeature-provider-flagd/pyproject.toml b/providers/openfeature-provider-flagd/pyproject.toml index 91f6e66f..df3163a7 100644 --- a/providers/openfeature-provider-flagd/pyproject.toml +++ b/providers/openfeature-provider-flagd/pyproject.toml @@ -35,6 +35,12 @@ dev = [ "coverage[toml]>=7.10.0,<8.0.0", "grpcio-health-checking>=1.82.1,<2.0.0", "mypy>=1.18.0,<2.0.0", + # The OpenFeature conformance suite. Ships the feature files, the flag set, + # the control-API client and the Compose harness that owns the container + # stack, and registers its step definitions through a pytest11 entry point -- + # so tests/tck declares a Compose file and nothing else. The `compose` extra + # is what pulls testcontainers in for the harness. + "openfeature-tck[compose]", "poethepoet>=0.37.0", "pytest>=9.0.0,<10.0.0", "pytest-bdd>=8.1.0,<9.0.0", @@ -112,13 +118,58 @@ warn_unused_ignores = false [tool.uv.sources] openfeature-flagd-core = { workspace = true } +openfeature-tck = { workspace = true } [tool.pytest] strict = true [tool.poe.tasks] -test = "pytest tests" -test-cov = "coverage run -m pytest tests" +# `tests/tck` is excluded from the default build on purpose, and a maintainer +# runs `poe test-tck` by hand before merging a change to it. Why a conformance +# suite is not a required gate is Appendix F, "Running the suite in CI" -- +# linked from tools/openfeature-tck/README.md -- and is deliberately not +# restated here. +# +# What is local to this package: a full run is 8 failed, 119 passed, 3 skipped. +# Six of those failures are canonical flags flagd-testbed does not seed yet, on +# each of the two resolvers; the other two are real provider-side gaps, one per +# resolver. tests/tck/conftest.py accounts for all eight and for the three +# skips. Docker is not what decides the exclusion -- `tests/e2e` needs Docker +# too and does run here. +# +# The suite is still *collected* on every default build, so it cannot quietly +# stop importing against the harness while nobody runs it. `--ignore` would +# otherwise leave nothing checking that, since mypy here is configured over +# `src` alone. +# +# `ignore_fail = "return_non_zero"` is what makes that true rather than +# aspirational: poe aborts a sequence at its first failing subtask, and this +# package's default suite is red on this branch -- the TCK requires +# openfeature-sdk >= 0.10 and this provider is not updated for it yet -- so the +# collect step sat behind a step that never succeeds and never ran. Every +# subtask now runs and a non-zero exit still propagates. +test = { sequence = ["test-default", "test-tck-collect"], ignore_fail = "return_non_zero" } +test-cov = { sequence = ["test-cov-default", "test-tck-collect"], ignore_fail = "return_non_zero" } +test-default = "pytest tests --ignore=tests/tck" +test-cov-default = "coverage run -m pytest tests --ignore=tests/tck" +# The sync is part of the run rather than something a maintainer remembers, +# because Appendix F requires the revision check to be in force **where the +# scenarios execute** and not only in the TCK implementation's own tests. This +# is a uv workspace, so `openfeature-tck` is installed editable and the +# canonical assets this suite collects are the gitignored copies under +# `tools/openfeature-tck/src`: a rebase that moves the submodule pin leaves them +# from the previous revision, and a suite run from here would neither notice nor +# say so. That is exactly how an adoption in another language published a full +# pass against stale scenarios. `hatch_build_sync.py` checks the submodule out +# at the pin, refuses to copy if it cannot, and needs nothing but the standard +# library. +# +# Not on `test-tck-collect`, deliberately: that runs on every default build, it +# executes no scenario, and it publishes no result, so the rule does not reach +# it and the default build stays free of a submodule operation. +sync-spec-assets = "python ../../tools/openfeature-tck/hatch_build_sync.py" +test-tck = ["sync-spec-assets", {cmd = "pytest tests/tck"}] +test-tck-collect = "pytest tests/tck --collect-only -q" cov-report = "coverage xml" cov = [ "test-cov", diff --git a/providers/openfeature-provider-flagd/tests/tck/__init__.py b/providers/openfeature-provider-flagd/tests/tck/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py new file mode 100644 index 00000000..556d8191 --- /dev/null +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -0,0 +1,69 @@ +"""One testbed stack, declared rather than wired, and shared by both suites. + +One flagd process serves both resolver ports -- 8013 for RPC, 8015 for sync -- +so one stack and one ``HttpControl`` serve both suites. + +**A full run is ``8 failed, 119 passed, 3 skipped``.** All eight failures have +one of three causes, and only two of them are the provider's: + +* Six are the backend's flag set. No released flagd-testbed seeds + ``large-integer-flag`` or ``integral-float-flag``, so three scenarios fail + ``FLAG_NOT_FOUND`` on each resolver alike. open-feature/flagd-testbed#392 + seeds them; bump the tag in ``docker-compose.yaml`` beside this file when it + lands. Carrying no ``KnownDeviation``: the provider was never given the flag. +* One is ``openfeature-flagd-core``'s, on in-process alone. ``boolean-flag`` + requested as a Float resolves to ``1.0`` rather than the caller's default, + because ``bool`` is a subclass of ``int``: python-sdk-contrib#417. RPC passes + the row, because the server type-checks it. +* One is flagd's, on RPC alone, and is the only failure here carrying a + ``KnownDeviation``: ``float-flag`` (0.5) as an Integer comes back as ``0``. + In-process passes that scenario, hence the deviation on RPC only. + +The three skips are two scenarios: ``@large-integers``, withheld on both +resolvers, and ``@reinitialization``, withheld on RPC alone. + +One finding has nowhere else to go, because its scenario *passes*: in-process +shutdown-under-outage leaves a ``PytestUnhandledThreadExceptionWarning`` from +gRPC's connectivity thread behind it on every run, python-sdk-contrib#419. Noted +so a reader knows the warning is recorded rather than noise. +""" + +from __future__ import annotations + +import socket +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.tck import ComposeBackend, RunningBackend +from tests.tck.suite import IN_PROCESS_PORT, RPC_PORT + + +@pytest.fixture(scope="session") +def compose_backend() -> ComposeBackend: + """The stack under test, for the TCK's ``tck_backend`` fixture. + + Both resolver ports are declared even though each suite uses one of them, + because both suites share this stack. + + The path is absolute so that pytest run from the repository root works too; + a relative one resolves against the working directory. + """ + return ComposeBackend( + compose_file=Path(__file__).parent / "docker-compose.yaml", + backend_ports=[RPC_PORT, IN_PROCESS_PORT], + ) + + +@pytest.fixture(scope="session") +def closed_port(tck_backend: RunningBackend) -> int: + """A port on localhost with nothing listening, for the ``@unavailable`` scenarios. + + Discovered by binding and releasing rather than hard-coded, because the + stack's own host ports are mapped dynamically and a fixed number could + collide with one; depending on ``tck_backend`` orders this after the stack + has taken its ports. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + return int(probe.getsockname()[1]) diff --git a/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml b/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml new file mode 100644 index 00000000..a230ea66 --- /dev/null +++ b/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml @@ -0,0 +1,20 @@ +# The backend every conformance suite in this repository runs against: the unmodified +# flagd testbed image, serving flagd and its launchpad control API. +# +# Not flagd-testbed's own compose file, which adds an envoy sidecar for its +# forbidden-endpoint scenarios, names its service `flagd`, and bind-mounts a flags +# directory the launchpad writes into. None of that is needed to drive the launchpad. +# +# This file exists twice, once per provider package, and the two are byte-identical: +# providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml +# providers/openfeature-provider-ofrep/tests/tck/docker-compose.yaml +# Each package publishes its own distribution and must not read the other's files, so +# `diff` the two paths after changing either -- that diff is what catches drift. +services: + backend: + image: ghcr.io/open-feature/flagd-testbed:v3.10.3 + ports: + - 8013 # flagd RPC evaluation (gRPC) + - 8015 # flagd in-process sync (gRPC) + - 8016 # flagd's OFREP HTTP API + - 8080 # launchpad control API diff --git a/providers/openfeature-provider-flagd/tests/tck/suite.py b/providers/openfeature-provider-flagd/tests/tck/suite.py new file mode 100644 index 00000000..6d43bcec --- /dev/null +++ b/providers/openfeature-provider-flagd/tests/tck/suite.py @@ -0,0 +1,176 @@ +"""Shared wiring for the two flagd conformance suites. + +flagd resolves flags two quite different ways -- RPC evaluates remotely over +gRPC, in-process syncs the ruleset and evaluates locally -- and they are +separate suites because they are separately conformant. What is shared is the +timings, which are flagd's own and interact, and the one function that turns a +resolver plus a running endpoint into a ``TckConfig``; what differs lives in +``test_rpc.py`` and ``test_in_process.py``. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from openfeature.contrib.provider.flagd import FlagdProvider +from openfeature.contrib.provider.flagd.config import ResolverType +from openfeature.contrib.tools.tck import ( + Capability, + KnownDeviation, + RunningBackend, + TckConfig, +) +from openfeature.provider import FeatureProvider + +__all__ = ["IN_PROCESS_PORT", "RPC_PORT", "ResolverSuite", "build_config"] + +RPC_PORT = 8013 +IN_PROCESS_PORT = 8015 +"""Container-internal ports flagd serves remote evaluation and the sync stream on. + +Named here rather than in ``conftest.py`` so that the declaration and the +resolver that uses it cannot drift apart. +""" + +DEADLINE_MS = 5000 +"""How long a provider blocks during initialisation before giving up. + +This, and not ``TckConfig.ready_timeout``, is what actually bounds flagd's +initialisation: the resolvers block inside ``initialize`` and raise +``ProviderNotReadyError`` when it expires (grpc.py:175, grpc_watcher.py:151). +Generous because every scenario is preceded by a control-API ``/start``, so a +provider is routinely built against a backend that came up milliseconds ago. + +For the RPC resolver it also bounds each individual resolution call. +""" + +RETRY_BACKOFF_MS = 500 +"""Initial delay before a reconnect attempt. Short, so an ended outage is noticed quickly.""" + +RETRY_BACKOFF_MAX_MS = 5000 +"""Longest delay between reconnect attempts, and the wait between stream retries. + +Kept at or above :data:`DEADLINE_MS`: the provider passes the deadline to gRPC +as ``grpc.min_reconnect_backoff_ms`` and this as +``grpc.max_reconnect_backoff_ms`` (grpc.py:90-92), so a smaller value here would +be a channel configured with a minimum backoff above its maximum. +""" + +RETRY_GRACE_PERIOD_SECONDS = 30 +"""How long a disconnected provider stays STALE before escalating to ERROR. + +Load-bearing for the ``@stale`` scenario: both resolvers start the escalation +timer the moment the channel fails (grpc.py:204-210, grpc_watcher.py:180-186), +so too short a value turns a scenario about staleness into one about failure. +""" + +STREAM_DEADLINE_MS = 0 +"""Disable periodic stream recycling, as the existing e2e suites do. + +It removes a source of background reconnects from a suite whose whole subject +is what a reconnect looks like. +""" + +UNAVAILABLE_DEADLINE_MS = 500 +UNAVAILABLE_GRACE_PERIOD_SECONDS = 1 +UNAVAILABLE_BACKOFF_MS = 5000 +"""Deliberately impatient settings for the provider that cannot reach a backend. + +The ``@unavailable`` scenarios assert that failure is reported *promptly*, so a +provider taking 30 seconds to give up would pass a test about eventual failure +while failing the one that matters. The backoff is long for the opposite +reason: after the error is reported, the next failed reconnect attempt would +emit another ``PROVIDER_STALE`` and move the provider out of the ERROR state the +scenario is about to assert. +""" + +EVENT_TIMEOUT = 20.0 +"""Seconds to wait for a provider event. + +Comfortably above :data:`RETRY_BACKOFF_MAX_MS`, which is how long a provider may +wait before the reconnect attempt that produces the ``PROVIDER_READY`` ending +the ``@stale`` scenario. +""" + + +@dataclass(frozen=True) +class ResolverSuite: + """What differs between the two resolvers' suites.""" + + name: str + """Names the suite in test output and scopes its OpenFeature domain.""" + + resolver_type: ResolverType + + backend_port: int + """Container-internal port, resolved to a host port once the stack is up.""" + + capabilities: frozenset[Capability] + """What this resolver was run against the suite and seen to satisfy. + + See each suite module for the evidence behind every entry and omission. + """ + + ready_timeout: float + + known_deviations: tuple[KnownDeviation, ...] = () + """Requirements this resolver is known to fail, acknowledged rather than hidden. + + **Per resolver, and that is the whole reason this field is here rather than + shared.** RPC narrows ``float-flag`` to ``0`` where in-process refuses it, + so an entry naming :attr:`~Capability.NUMERIC_COERCION` is true of one and + false of the other, and attaching it to both would publish a defect against + the resolver that does not have it. + """ + + +def build_config( + suite: ResolverSuite, + backend: RunningBackend, + closed_port: int, +) -> TckConfig: + """Wire one resolver up to the running testbed. + + A factory because the mapped host port does not exist until the stack is up. + The host comes from the endpoint rather than being written as + ``"localhost"``, because with a remote Docker daemon, Docker Desktop on some + platforms or a rootless setup it is neither localhost nor predictable. + """ + endpoint = backend.endpoint + + def new_provider() -> FeatureProvider: + return FlagdProvider( + resolver_type=suite.resolver_type, + host=endpoint.host, + port=endpoint.port(suite.backend_port), + deadline_ms=DEADLINE_MS, + stream_deadline_ms=STREAM_DEADLINE_MS, + retry_backoff_ms=RETRY_BACKOFF_MS, + retry_backoff_max_ms=RETRY_BACKOFF_MAX_MS, + retry_grace_period=RETRY_GRACE_PERIOD_SECONDS, + ) + + def new_unavailable_provider() -> FeatureProvider: + # A closed port on localhost, never the backend under test: that has to + # stay up for every other scenario. + return FlagdProvider( + resolver_type=suite.resolver_type, + host="localhost", + port=closed_port, + deadline_ms=UNAVAILABLE_DEADLINE_MS, + stream_deadline_ms=STREAM_DEADLINE_MS, + retry_backoff_ms=UNAVAILABLE_BACKOFF_MS, + retry_backoff_max_ms=UNAVAILABLE_BACKOFF_MS, + retry_grace_period=UNAVAILABLE_GRACE_PERIOD_SECONDS, + ) + + return TckConfig( + name=suite.name, + control=backend.control, + new_provider=new_provider, + new_unavailable_provider=new_unavailable_provider, + capabilities=suite.capabilities, + known_deviations=suite.known_deviations, + event_timeout=EVENT_TIMEOUT, + ready_timeout=suite.ready_timeout, + ) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py new file mode 100644 index 00000000..89eefc29 --- /dev/null +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py @@ -0,0 +1,128 @@ +"""The OpenFeature provider conformance suite, run against flagd's in-process resolver. + +In-process syncs the whole ruleset over flagd's sync API and evaluates locally, +so unlike RPC the type-checking, the variant selection and the reason all come +from ``openfeature-flagd-core`` in this process rather than from the server -- +which is why this is a separate suite rather than a parametrisation of the RPC +one. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.provider.flagd.config import ResolverType +from openfeature.contrib.tools.tck import ( + Capability, + RunningBackend, + TckConfig, + feature_paths, +) +from tests.tck.suite import IN_PROCESS_PORT, ResolverSuite, build_config + +# Every capability below was declared, the suite run, and its scenarios seen to +# pass. The code references say where the behaviour lives, so a reader can check +# the claim. +# +# EVENTS grpc_watcher.py:262 emits PROVIDER_READY only once the first sync +# payload has been *applied* (grpc_watcher.py:254), so a scenario +# evaluating straight after ready cannot race the sync. +# STALE grpc_watcher.py:178-190 goes STALE on TRANSIENT_FAILURE and only +# then starts the timer escalating to ERROR. +# CONFIGURATION_CHANGE +# in_process.py:34 names exactly the keys FlagdCore reports as +# changed. +# OBJECT in_process.py:122. +# VARIANTS flagd_core.py returns the selected variant with every resolution; +# in_process.py carries it into the details. +# TARGETING targeting.py:40-41 puts the targeting key into the JSON-logic +# context; flagd_core.py:154 evaluates the rule against it. +# UNAVAILABLE_INIT +# grpc_watcher.py:151 raises ProviderNotReadyError once the +# blocking init deadline passes without a synced ruleset. +# DISABLED_FLAGS +# flagd_core.py:143-145 returns the caller's `default_value` with +# reason DISABLED before targeting or variant selection, and +# flagd_core.py:199-200 then skips the type check for that reason +# -- which is what stops the substituted default being re-typed +# against the flag it did not come from. +# LIFECYCLE Not a formality: this resolver applies the entire ruleset before +# ready (grpc_watcher.py:254-262) and initialisation can fail +# (grpc_watcher.py:151), so both terminal outcomes are reachable. +# STANDARD_REASONS +# All nine scenarios pass; the last three also need @targeting and +# @disabled-flags, both declared here. +# REINITIALIZATION +# Withheld on RPC, declared here on the same run: this resolver +# restarts serving `boolean-flag` correctly where RPC evaluates +# against a closed channel. +# +# NUMERIC_COERCION +# Declared on RPC too, but not the same claim: **this resolver satisfies the +# lossy half and RPC does not.** Evaluation is local, so flagd_core.py:25 +# admits only `int` for an integer request and `_check_type` +# (flagd_core.py:228-231) raises TypeMismatchError for anything else, while +# the float mapping at flagd_core.py:26 is the wider `(int, float)`. No +# KnownDeviation here, and RPC's entry is deliberately not mirrored onto +# this suite (open-feature/python-sdk-contrib#420). The tag's third scenario +# fails for a reason that is not about coercion: flagd-testbed seeds no +# `integral-float-flag`. +# +# STRING_TYPING and FULLY_TYPED_VALUES +# Both declared, on a run. `_TYPE_MAP`'s string entry is the narrow `(str,)` +# (flagd_core.py:23) and `_check_type` (flagd_core.py:228-231) raises +# TypeMismatchError for anything else, so nothing renders a bool, a number +# or a structure as text -- the one accessor `bool` does not slip through, +# because `str` is not in bool's ancestry. The ruleset is `json.loads` +# output all the way down (flagd_core.py:73), so there is no type this +# resolver keeps as text and the second tag has an answer here too. +# +# Not declared, and why: +# +# LARGE_INTEGERS +# Withheld: the tag's one scenario asks for `huge-integer-flag` and no +# released flagd-testbed seeds it, so none of its scenarios reach this +# provider. No KnownDeviation -- the gap is the backend's. Declare the tag +# once open-feature/flagd-testbed#392 is in the image, or the withholding +# starts reading as a claim about the provider. +# +# CACHING +# No scenario carries the tag. +IN_PROCESS_CAPABILITIES = frozenset( + { + Capability.EVENTS, + Capability.STALE, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.VARIANTS, + Capability.DISABLED_FLAGS, + Capability.TARGETING, + Capability.UNAVAILABLE_INIT, + Capability.STANDARD_REASONS, + Capability.LIFECYCLE, + Capability.NUMERIC_COERCION, + Capability.STRING_TYPING, + Capability.FULLY_TYPED_VALUES, + Capability.REINITIALIZATION, + } +) + +IN_PROCESS_SUITE = ResolverSuite( + name="flagd-in-process", + resolver_type=ResolverType.IN_PROCESS, + backend_port=IN_PROCESS_PORT, + capabilities=IN_PROCESS_CAPABILITIES, + # In-process transfers and applies the whole ruleset before reporting ready, + # so it needs more headroom than RPC. + ready_timeout=60.0, +) + + +@pytest.fixture(scope="session") +def tck_config(tck_backend: RunningBackend, closed_port: int) -> TckConfig: + """The whole of this adoption's wiring.""" + return build_config(IN_PROCESS_SUITE, tck_backend, closed_port) + + +scenarios(*feature_paths()) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py new file mode 100644 index 00000000..b1333afc --- /dev/null +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py @@ -0,0 +1,167 @@ +"""The OpenFeature provider conformance suite, run against flagd's RPC resolver. + +RPC asks flagd to evaluate each flag over gRPC and maps the response onto typed +resolution details, so what is under test here is that mapping plus the +lifecycle the evaluation stream drives. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.provider.flagd.config import ResolverType +from openfeature.contrib.tools.tck import ( + Capability, + KnownDeviation, + RunningBackend, + TckConfig, + feature_paths, +) +from tests.tck.suite import RPC_PORT, ResolverSuite, build_config + +# Every capability below was declared, the suite run, and its scenarios seen to +# pass. The code references say where the behaviour lives, so a reader can check +# the claim. +# +# EVENTS grpc.py:261 emits PROVIDER_READY when the evaluation stream +# delivers its 'provider_ready' message. +# STALE grpc.py:202-212 goes STALE on TRANSIENT_FAILURE and only then +# starts the timer escalating to ERROR. +# CONFIGURATION_CHANGE +# grpc.py:302 names the changed keys, and grpc.py:298-300 evicts +# exactly those from the LRU cache -- so the re-evaluation the +# scenario performs cannot be served a stale cached value. +# OBJECT grpc.py:336, through ResolveObject. +# VARIANTS grpc.py:449 carries the response's `variant` into the details. +# Seven of the outline's eight rows pass; the eighth asks for a +# flag the backend does not seed. +# TARGETING grpc.py:492 puts the targeting key into the request's context +# struct, so the server evaluates the rule against it. +# UNAVAILABLE_INIT +# grpc.py:175 raises ProviderNotReadyError once the blocking init +# deadline passes without a connection. +# DISABLED_FLAGS +# All four rows pass, which is not obvious for a remote evaluator: +# flagd sends reason DISABLED, no variant and the proto's zero +# value, and grpc.py:468-472 substitutes the caller's +# `default_value` locally. What crosses the wire is the signal. +# LIFECYCLE Not a formality: this resolver blocks until the evaluation stream +# is up and raises ProviderNotReadyError when the deadline passes +# (grpc.py:175, grpc.py:261), so both terminal outcomes are +# reachable. Five of six pass; the sixth is @reinitialization, +# below. +# STANDARD_REASONS +# All nine scenarios pass; the last three also need @targeting and +# @disabled-flags, both declared here. The DISABLED row passes +# despite the reason arriving as flagd's bare string rather than +# the SDK's Reason enum, because the step compares it as text. +# +# NUMERIC_COERCION +# Declared, and failing the lossy scenario deliberately -- the failure is the +# report; see KNOWN_DEVIATIONS below. RPC does not type-check locally: +# grpc.py:444-448 asks flagd for an Int and passes back whatever the server +# answers, and flagd's evaluator casts the float64 variant with a bare +# `int64(val)` (core/pkg/evaluator/json.go, ResolveIntValue), so +# `float-flag`'s 0.5 comes back as 0 with no error code. +# +# **The in-process resolver passes that scenario**, refusing 0.5 as an +# Integer, which is why the deviation is on this suite only and why the tag +# is declared and left to fail rather than withheld: a skip cannot say "it +# coerces, and one direction is wrong" (python-sdk-contrib#420). The tag's +# third scenario fails for a reason that is not flagd's: flagd-testbed seeds +# no `integral-float-flag`. +# +# STRING_TYPING and FULLY_TYPED_VALUES +# Both declared, on a run. The decision is flagd's for the same reason the +# coercion one is -- grpc.py asks the server for a String and hands back +# what it answers -- and flagd holds a ruleset rather than a string table, +# typing every variant out of the JSON, so a bool, a number or a structure +# reached through ResolveString is a TYPE_MISMATCH the server raises. That +# is not a property of flagd that could be read off this repository at all, +# which is why both tags rest on a run rather than on an argument. +# +# Not declared, and why: +# +# LARGE_INTEGERS +# Withheld: the tag's one scenario asks for `huge-integer-flag` and no +# released flagd-testbed seeds it, so none of its scenarios reach this +# provider. Nothing in this path would narrow the value -- flagd holds every +# numeric variant as a float64, 2^53 - 1 is exactly the largest integer a +# float64 represents without rounding, and grpc.py:448 hands +# ResolveIntResponse.value to the SDK as an unbounded Python int -- and the +# suite cannot show that, which is the point. No KnownDeviation: the gap is +# the backend's. Declare the tag once open-feature/flagd-testbed#392 is in +# the image, or the withholding starts reading as a claim about the provider. +# +# REINITIALIZATION +# Withheld, on the measurement: with LIFECYCLE declared the scenario runs +# and fails, because grpc.py:420 raises "Cannot invoke RPC on closed +# channel!" -- shutdown() closes the channel and the second initialize() +# does not rebuild it. No KnownDeviation: requirement 2.5.2 permits reuse +# rather than requiring it. The in-process suite declares it for the same +# reason in reverse -- it runs, and it passes. +# +# CACHING +# No scenario carries the tag. +RPC_CAPABILITIES = frozenset( + { + Capability.EVENTS, + Capability.STALE, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.VARIANTS, + Capability.DISABLED_FLAGS, + Capability.TARGETING, + Capability.UNAVAILABLE_INIT, + Capability.STANDARD_REASONS, + Capability.LIFECYCLE, + Capability.NUMERIC_COERCION, + Capability.STRING_TYPING, + Capability.FULLY_TYPED_VALUES, + } +) + +KNOWN_DEVIATIONS = ( + KnownDeviation.tracked( + capability=Capability.NUMERIC_COERCION, + issue="https://github.com/open-feature/flagd/issues/1996", + summary=( + "The lossy half of the coercion rule is not enforced: evaluating " + "float-flag (0.5) through the integer accessor returns 0 with no " + "error code, rather than TYPE_MISMATCH with the code default, so " + "the fractional part is discarded silently. Lossless coercion is " + "permitted and is not the defect -- this resolver does widen an " + "integer to a float correctly, which is why the capability is " + "declared and the scenario left to fail rather than the capability " + "withheld. The defect is in flagd itself rather than in this " + "provider layer: the Python in-process resolver evaluates locally " + "and refuses 0.5 as an integer correctly, so it is not recorded as " + "deviating and carries no equivalent entry. The tag's third " + "scenario also fails, but for an unrelated reason that is not " + "flagd's: integral-float-flag is absent from the pinned " + "flagd-testbed image, open-feature/flagd-testbed#392." + ), + ), +) +"""The one requirement this resolver is known to fail; the in-process suite passes it.""" + +RPC_SUITE = ResolverSuite( + name="flagd-rpc", + resolver_type=ResolverType.RPC, + backend_port=RPC_PORT, + capabilities=RPC_CAPABILITIES, + known_deviations=KNOWN_DEVIATIONS, + # RPC holds no ruleset of its own: it is ready as soon as the evaluation + # stream is up, so it needs less headroom than in-process. + ready_timeout=30.0, +) + + +@pytest.fixture(scope="session") +def tck_config(tck_backend: RunningBackend, closed_port: int) -> TckConfig: + """The whole of this adoption's wiring.""" + return build_config(RPC_SUITE, tck_backend, closed_port) + + +scenarios(*feature_paths()) diff --git a/uv.lock b/uv.lock index 4d26a18b..c53f6392 100644 --- a/uv.lock +++ b/uv.lock @@ -1932,6 +1932,7 @@ dev = [ { name = "coverage", extra = ["toml"] }, { name = "grpcio-health-checking" }, { name = "mypy" }, + { name = "openfeature-tck", extra = ["compose"] }, { name = "poethepoet" }, { name = "pytest" }, { name = "pytest-bdd" }, @@ -1957,6 +1958,7 @@ dev = [ { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, { name = "grpcio-health-checking", specifier = ">=1.82.1,<2.0.0" }, { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, + { name = "openfeature-tck", extras = ["compose"], editable = "tools/openfeature-tck" }, { name = "poethepoet", specifier = ">=0.37.0" }, { name = "pytest", specifier = ">=9.0.0,<10.0.0" }, { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" },