test(flagd): run the provider conformance suite against both resolvers

providers/openfeature-provider-flagd/README.md

@@ -190,6 +190,46 @@ api.set_provider(
 )
 ```
 
+## Provider conformance suite
+
+This provider runs the [OpenFeature Provider Conformance Suite][tck] against a flagd-testbed stack,
+once per resolver, in `tests/tck`. The suite owns the container stack: `tests/tck/conftest.py`
+declares a Compose file and the two ports the resolvers connect to, and nothing else.
+
+`tests/tck/docker-compose.yaml` is one definition of the backend for the whole repository, and the
+OFREP adoption carries a byte-identical copy — each provider package publishes its own distribution
+and must not read the other's files, so the two are kept in step by `diff` rather than by sharing a
+path. Change one, copy it to the other.
+
+**It is excluded from the default build, and a maintainer runs it by hand before merging a change to
+it.**
+
+```
+poe test-tck        # both resolvers, needs Docker
+poe test            # everything else, which is what CI runs
+```
+
+The exclusion lives in `pyproject.toml`: `--ignore=tests/tck` on the two tasks `build.yml` reaches,
+with the reason in a comment above them. Why a conformance suite is not a required gate is
+[Appendix F, "Running the suite in CI"][appendix-f], and is not restated here.
+
+Two things that are this provider's rather than the policy's:
+
+- **Docker is not what decides it.** `tests/e2e` needs Docker too and does run in the default build.
+  What decides it is the run: **8 failed, 119 passed, 3 skipped**, being three canonical flags that
+  no released flagd-testbed seeds yet, failing on each resolver, plus two genuine provider-side
+  gaps — one per resolver, and only one of them carries a known-deviation entry.
+  `tests/tck/conftest.py` accounts for all eight individually, with the flag or requirement each one
+  turns on, so a reviewer running the suite can tell a new failure from a known one and the number
+  above is what to expect.
+- **The default build still collects the suite** — `poe test` and `poe test-cov` end in
+  `pytest tests/tck --collect-only`, which imports every module and starts no container. An excluded
+  suite that has quietly stopped importing against the harness is worse than one that runs and
+  fails, and `mypy` here is configured over `src` alone, so nothing else would notice.
+
+[tck]: ../../tools/openfeature-tck/README.md
+[appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md
+
 ## License
 
 Apache 2.0 - See [LICENSE](./LICENSE) for more information.

providers/openfeature-provider-flagd/pyproject.toml

@@ -35,6 +35,12 @@ dev = [
   "coverage[toml]>=7.10.0,<8.0.0",
   "grpcio-health-checking>=1.82.1,<2.0.0",
   "mypy>=1.18.0,<2.0.0",
+  # The OpenFeature conformance suite. Ships the feature files, the flag set,
+  # the control-API client and the Compose harness that owns the container
+  # stack, and registers its step definitions through a pytest11 entry point --
+  # so tests/tck declares a Compose file and nothing else. The `compose` extra
+  # is what pulls testcontainers in for the harness.
+  "openfeature-tck[compose]",
   "poethepoet>=0.37.0",
   "pytest>=9.0.0,<10.0.0",
   "pytest-bdd>=8.1.0,<9.0.0",
@@ -112,13 +118,58 @@ warn_unused_ignores = false
 
 [tool.uv.sources]
 openfeature-flagd-core = { workspace = true }
+openfeature-tck = { workspace = true }
 
 [tool.pytest]
 strict = true
 
 [tool.poe.tasks]
-test = "pytest tests"
-test-cov = "coverage run -m pytest tests"
+# `tests/tck` is excluded from the default build on purpose, and a maintainer
+# runs `poe test-tck` by hand before merging a change to it. Why a conformance
+# suite is not a required gate is Appendix F, "Running the suite in CI" --
+# linked from tools/openfeature-tck/README.md -- and is deliberately not
+# restated here.
+#
+# What is local to this package: a full run is 8 failed, 119 passed, 3 skipped.
+# Six of those failures are canonical flags flagd-testbed does not seed yet, on
+# each of the two resolvers; the other two are real provider-side gaps, one per
+# resolver. tests/tck/conftest.py accounts for all eight and for the three
+# skips. Docker is not what decides the exclusion -- `tests/e2e` needs Docker
+# too and does run here.
+#
+# The suite is still *collected* on every default build, so it cannot quietly
+# stop importing against the harness while nobody runs it. `--ignore` would
+# otherwise leave nothing checking that, since mypy here is configured over
+# `src` alone.
+#
+# `ignore_fail = "return_non_zero"` is what makes that true rather than
+# aspirational: poe aborts a sequence at its first failing subtask, and this
+# package's default suite is red on this branch -- the TCK requires
+# openfeature-sdk >= 0.10 and this provider is not updated for it yet -- so the
+# collect step sat behind a step that never succeeds and never ran. Every
+# subtask now runs and a non-zero exit still propagates.
+test = { sequence = ["test-default", "test-tck-collect"], ignore_fail = "return_non_zero" }
+test-cov = { sequence = ["test-cov-default", "test-tck-collect"], ignore_fail = "return_non_zero" }
+test-default = "pytest tests --ignore=tests/tck"
+test-cov-default = "coverage run -m pytest tests --ignore=tests/tck"
+# The sync is part of the run rather than something a maintainer remembers,
+# because Appendix F requires the revision check to be in force **where the
+# scenarios execute** and not only in the TCK implementation's own tests. This
+# is a uv workspace, so `openfeature-tck` is installed editable and the
+# canonical assets this suite collects are the gitignored copies under
+# `tools/openfeature-tck/src`: a rebase that moves the submodule pin leaves them
+# from the previous revision, and a suite run from here would neither notice nor
+# say so. That is exactly how an adoption in another language published a full
+# pass against stale scenarios. `hatch_build_sync.py` checks the submodule out
+# at the pin, refuses to copy if it cannot, and needs nothing but the standard
+# library.
+#
+# Not on `test-tck-collect`, deliberately: that runs on every default build, it
+# executes no scenario, and it publishes no result, so the rule does not reach
+# it and the default build stays free of a submodule operation.
+sync-spec-assets = "python ../../tools/openfeature-tck/hatch_build_sync.py"
+test-tck = ["sync-spec-assets", {cmd = "pytest tests/tck"}]
+test-tck-collect = "pytest tests/tck --collect-only -q"
 cov-report = "coverage xml"
 cov = [
     "test-cov",

providers/openfeature-provider-flagd/tests/tck/conftest.py

@@ -0,0 +1,69 @@
+"""One testbed stack, declared rather than wired, and shared by both suites.
+
+One flagd process serves both resolver ports -- 8013 for RPC, 8015 for sync --
+so one stack and one ``HttpControl`` serve both suites.
+
+**A full run is ``8 failed, 119 passed, 3 skipped``.** All eight failures have
+one of three causes, and only two of them are the provider's:
+
+* Six are the backend's flag set. No released flagd-testbed seeds
+  ``large-integer-flag`` or ``integral-float-flag``, so three scenarios fail
+  ``FLAG_NOT_FOUND`` on each resolver alike. open-feature/flagd-testbed#392
+  seeds them; bump the tag in ``docker-compose.yaml`` beside this file when it
+  lands. Carrying no ``KnownDeviation``: the provider was never given the flag.
+* One is ``openfeature-flagd-core``'s, on in-process alone. ``boolean-flag``
+  requested as a Float resolves to ``1.0`` rather than the caller's default,
+  because ``bool`` is a subclass of ``int``: python-sdk-contrib#417. RPC passes
+  the row, because the server type-checks it.
+* One is flagd's, on RPC alone, and is the only failure here carrying a
+  ``KnownDeviation``: ``float-flag`` (0.5) as an Integer comes back as ``0``.
+  In-process passes that scenario, hence the deviation on RPC only.
+
+The three skips are two scenarios: ``@large-integers``, withheld on both
+resolvers, and ``@reinitialization``, withheld on RPC alone.
+
+One finding has nowhere else to go, because its scenario *passes*: in-process
+shutdown-under-outage leaves a ``PytestUnhandledThreadExceptionWarning`` from
+gRPC's connectivity thread behind it on every run, python-sdk-contrib#419. Noted
+so a reader knows the warning is recorded rather than noise.
+"""
+
+from __future__ import annotations
+
+import socket
+from pathlib import Path
+
+import pytest
+
+from openfeature.contrib.tools.tck import ComposeBackend, RunningBackend
+from tests.tck.suite import IN_PROCESS_PORT, RPC_PORT
+
+
+@pytest.fixture(scope="session")
+def compose_backend() -> ComposeBackend:
+    """The stack under test, for the TCK's ``tck_backend`` fixture.
+
+    Both resolver ports are declared even though each suite uses one of them,
+    because both suites share this stack.
+
+    The path is absolute so that pytest run from the repository root works too;
+    a relative one resolves against the working directory.
+    """
+    return ComposeBackend(
+        compose_file=Path(__file__).parent / "docker-compose.yaml",
+        backend_ports=[RPC_PORT, IN_PROCESS_PORT],
+    )
+
+
+@pytest.fixture(scope="session")
+def closed_port(tck_backend: RunningBackend) -> int:
+    """A port on localhost with nothing listening, for the ``@unavailable`` scenarios.
+
+    Discovered by binding and releasing rather than hard-coded, because the
+    stack's own host ports are mapped dynamically and a fixed number could
+    collide with one; depending on ``tck_backend`` orders this after the stack
+    has taken its ports.
+    """
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
+        probe.bind(("127.0.0.1", 0))
+        return int(probe.getsockname()[1])

providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml

@@ -0,0 +1,20 @@
+# The backend every conformance suite in this repository runs against: the unmodified
+# flagd testbed image, serving flagd and its launchpad control API.
+#
+# Not flagd-testbed's own compose file, which adds an envoy sidecar for its
+# forbidden-endpoint scenarios, names its service `flagd`, and bind-mounts a flags
+# directory the launchpad writes into. None of that is needed to drive the launchpad.
+#
+# This file exists twice, once per provider package, and the two are byte-identical:
+#   providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml
+#   providers/openfeature-provider-ofrep/tests/tck/docker-compose.yaml
+# Each package publishes its own distribution and must not read the other's files, so
+# `diff` the two paths after changing either -- that diff is what catches drift.
+services:
+  backend:
+    image: ghcr.io/open-feature/flagd-testbed:v3.10.3
+    ports:
+      - 8013   # flagd RPC evaluation (gRPC)
+      - 8015   # flagd in-process sync (gRPC)
+      - 8016   # flagd's OFREP HTTP API
+      - 8080   # launchpad control API

providers/openfeature-provider-flagd/tests/tck/suite.py

@@ -0,0 +1,176 @@
+"""Shared wiring for the two flagd conformance suites.
+
+flagd resolves flags two quite different ways -- RPC evaluates remotely over
+gRPC, in-process syncs the ruleset and evaluates locally -- and they are
+separate suites because they are separately conformant. What is shared is the
+timings, which are flagd's own and interact, and the one function that turns a
+resolver plus a running endpoint into a ``TckConfig``; what differs lives in
+``test_rpc.py`` and ``test_in_process.py``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from openfeature.contrib.provider.flagd import FlagdProvider
+from openfeature.contrib.provider.flagd.config import ResolverType
+from openfeature.contrib.tools.tck import (
+    Capability,
+    KnownDeviation,
+    RunningBackend,
+    TckConfig,
+)
+from openfeature.provider import FeatureProvider
+
+__all__ = ["IN_PROCESS_PORT", "RPC_PORT", "ResolverSuite", "build_config"]
+
+RPC_PORT = 8013
+IN_PROCESS_PORT = 8015
+"""Container-internal ports flagd serves remote evaluation and the sync stream on.
+
+Named here rather than in ``conftest.py`` so that the declaration and the
+resolver that uses it cannot drift apart.
+"""
+
+DEADLINE_MS = 5000
+"""How long a provider blocks during initialisation before giving up.
+
+This, and not ``TckConfig.ready_timeout``, is what actually bounds flagd's
+initialisation: the resolvers block inside ``initialize`` and raise
+``ProviderNotReadyError`` when it expires (grpc.py:175, grpc_watcher.py:151).
+Generous because every scenario is preceded by a control-API ``/start``, so a
+provider is routinely built against a backend that came up milliseconds ago.
+
+For the RPC resolver it also bounds each individual resolution call.
+"""
+
+RETRY_BACKOFF_MS = 500
+"""Initial delay before a reconnect attempt. Short, so an ended outage is noticed quickly."""
+
+RETRY_BACKOFF_MAX_MS = 5000
+"""Longest delay between reconnect attempts, and the wait between stream retries.
+
+Kept at or above :data:`DEADLINE_MS`: the provider passes the deadline to gRPC
+as ``grpc.min_reconnect_backoff_ms`` and this as
+``grpc.max_reconnect_backoff_ms`` (grpc.py:90-92), so a smaller value here would
+be a channel configured with a minimum backoff above its maximum.
+"""
+
+RETRY_GRACE_PERIOD_SECONDS = 30
+"""How long a disconnected provider stays STALE before escalating to ERROR.
+
+Load-bearing for the ``@stale`` scenario: both resolvers start the escalation
+timer the moment the channel fails (grpc.py:204-210, grpc_watcher.py:180-186),
+so too short a value turns a scenario about staleness into one about failure.
+"""
+
+STREAM_DEADLINE_MS = 0
+"""Disable periodic stream recycling, as the existing e2e suites do.
+
+It removes a source of background reconnects from a suite whose whole subject
+is what a reconnect looks like.
+"""
+
+UNAVAILABLE_DEADLINE_MS = 500
+UNAVAILABLE_GRACE_PERIOD_SECONDS = 1
+UNAVAILABLE_BACKOFF_MS = 5000
+"""Deliberately impatient settings for the provider that cannot reach a backend.
+
+The ``@unavailable`` scenarios assert that failure is reported *promptly*, so a
+provider taking 30 seconds to give up would pass a test about eventual failure
+while failing the one that matters. The backoff is long for the opposite
+reason: after the error is reported, the next failed reconnect attempt would
+emit another ``PROVIDER_STALE`` and move the provider out of the ERROR state the
+scenario is about to assert.
+"""
+
+EVENT_TIMEOUT = 20.0
+"""Seconds to wait for a provider event.
+
+Comfortably above :data:`RETRY_BACKOFF_MAX_MS`, which is how long a provider may
+wait before the reconnect attempt that produces the ``PROVIDER_READY`` ending
+the ``@stale`` scenario.
+"""
+
+
+@dataclass(frozen=True)
+class ResolverSuite:
+    """What differs between the two resolvers' suites."""
+
+    name: str
+    """Names the suite in test output and scopes its OpenFeature domain."""
+
+    resolver_type: ResolverType
+
+    backend_port: int
+    """Container-internal port, resolved to a host port once the stack is up."""
+
+    capabilities: frozenset[Capability]
+    """What this resolver was run against the suite and seen to satisfy.
+
+    See each suite module for the evidence behind every entry and omission.
+    """
+
+    ready_timeout: float
+
+    known_deviations: tuple[KnownDeviation, ...] = ()
+    """Requirements this resolver is known to fail, acknowledged rather than hidden.
+
+    **Per resolver, and that is the whole reason this field is here rather than
+    shared.** RPC narrows ``float-flag`` to ``0`` where in-process refuses it,
+    so an entry naming :attr:`~Capability.NUMERIC_COERCION` is true of one and
+    false of the other, and attaching it to both would publish a defect against
+    the resolver that does not have it.
+    """
+
+
+def build_config(
+    suite: ResolverSuite,
+    backend: RunningBackend,
+    closed_port: int,
+) -> TckConfig:
+    """Wire one resolver up to the running testbed.
+
+    A factory because the mapped host port does not exist until the stack is up.
+    The host comes from the endpoint rather than being written as
+    ``"localhost"``, because with a remote Docker daemon, Docker Desktop on some
+    platforms or a rootless setup it is neither localhost nor predictable.
+    """
+    endpoint = backend.endpoint
+
+    def new_provider() -> FeatureProvider:
+        return FlagdProvider(
+            resolver_type=suite.resolver_type,
+            host=endpoint.host,
+            port=endpoint.port(suite.backend_port),
+            deadline_ms=DEADLINE_MS,
+            stream_deadline_ms=STREAM_DEADLINE_MS,
+            retry_backoff_ms=RETRY_BACKOFF_MS,
+            retry_backoff_max_ms=RETRY_BACKOFF_MAX_MS,
+            retry_grace_period=RETRY_GRACE_PERIOD_SECONDS,
+        )
+
+    def new_unavailable_provider() -> FeatureProvider:
+        # A closed port on localhost, never the backend under test: that has to
+        # stay up for every other scenario.
+        return FlagdProvider(
+            resolver_type=suite.resolver_type,
+            host="localhost",
+            port=closed_port,
+            deadline_ms=UNAVAILABLE_DEADLINE_MS,
+            stream_deadline_ms=STREAM_DEADLINE_MS,
+            retry_backoff_ms=UNAVAILABLE_BACKOFF_MS,
+            retry_backoff_max_ms=UNAVAILABLE_BACKOFF_MS,
+            retry_grace_period=UNAVAILABLE_GRACE_PERIOD_SECONDS,
+        )
+
+    return TckConfig(
+        name=suite.name,
+        control=backend.control,
+        new_provider=new_provider,
+        new_unavailable_provider=new_unavailable_provider,
+        capabilities=suite.capabilities,
+        known_deviations=suite.known_deviations,
+        event_timeout=EVENT_TIMEOUT,
+        ready_timeout=suite.ready_timeout,
+    )