test(flagd): run the provider conformance suite against both resolvers

providers/openfeature-provider-flagd/pyproject.toml

@@ -35,6 +35,10 @@ dev = [
   "coverage[toml]>=7.10.0,<8.0.0",
   "grpcio-health-checking>=1.82.1,<2.0.0",
   "mypy>=1.18.0,<2.0.0",
+  # The OpenFeature provider conformance suite. Ships the feature files, the flag
+  # set and the control-API client, and registers its step definitions through a
+  # pytest11 entry point, so tests/tck needs no conftest of its own for them.
+  "openfeature-provider-tck",
   "poethepoet>=0.37.0",
   "pytest>=9.0.0,<10.0.0",
   "pytest-bdd>=8.1.0,<9.0.0",
@@ -112,6 +116,7 @@ warn_unused_ignores = false
 
 [tool.uv.sources]
 openfeature-flagd-core = { workspace = true }
+openfeature-provider-tck = { workspace = true }
 
 [tool.pytest]
 strict = true

providers/openfeature-provider-flagd/tests/tck/conftest.py

@@ -0,0 +1,73 @@
+"""One testbed stack and one control, shared by both conformance suites.
+
+The stack is started once per session and **never restarted**. Scenario
+isolation comes from the control API instead, because container orchestrators
+assign host ports dynamically and cannot reliably preserve them across a
+restart: a restarted backend comes back on a different host port, silently
+invalidating every provider already pointed at the old one, and the failure
+looks like a flaky provider rather than a broken test. See the
+no-container-restart invariant in the TCK's ``control-api.yaml``.
+
+``flagd-testbed`` is not modified and the existing e2e suites are untouched: the
+TCK drives the testbed's launchpad through the standardised control API, which
+the launchpad already implements, and reuses the container lifecycle already in
+``tests/e2e``.
+"""
+
+from __future__ import annotations
+
+import socket
+import typing
+
+import pytest
+
+from openfeature.contrib.tools.provider_tck import HttpControl
+from tests.e2e.flagd_container import FlagdContainer
+
+
+@pytest.fixture(scope="session")
+def flagd_testbed() -> typing.Iterator[FlagdContainer]:
+    """The testbed stack, up for the whole session.
+
+    One stack for both suites because one flagd process serves both ports the
+    resolvers use -- 8013 for RPC and 8015 for sync -- so there is nothing a
+    second stack would isolate.
+    """
+    container = FlagdContainer()
+    container.start()
+    try:
+        yield container
+    finally:
+        container.stop()
+
+
+@pytest.fixture(scope="session")
+def flagd_control(flagd_testbed: FlagdContainer) -> HttpControl:
+    """The control API client, shared by both suites.
+
+    Shared rather than one per suite, and that matters: the two suites drive the
+    *same* backend, and ``HttpControl`` tracks whether a disconnect has left it
+    down so the next scenario starts it rather than merely resetting flag state.
+    Two instances would each hold half of that knowledge.
+
+    The launchpad registers only ``/start``, ``/restart``, ``/stop`` and
+    ``/change`` (flagd-testbed ``launchpad/main.go``), so ``/reset`` answers 404
+    and every ``prepare_scenario`` takes the documented ``/start`` fallback. The
+    probe costs one 404 for the whole session.
+    """
+    return HttpControl(flagd_testbed.get_launchpad_url())
+
+
+@pytest.fixture(scope="session")
+def closed_port(flagd_testbed: FlagdContainer) -> int:
+    """A port on localhost with nothing listening, for the ``@unavailable`` scenarios.
+
+    Discovered by binding and releasing rather than hard-coded, because the
+    testbed's own host ports are mapped dynamically and a hard-coded number
+    could collide with one. Depending on ``flagd_testbed`` orders this after the
+    stack has taken its ports, which is what makes the remaining race
+    negligible.
+    """
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
+        probe.bind(("127.0.0.1", 0))
+        return int(probe.getsockname()[1])

providers/openfeature-provider-flagd/tests/tck/suite.py

@@ -0,0 +1,167 @@
+"""Shared wiring for the two flagd conformance suites.
+
+flagd resolves flags two quite different ways -- RPC evaluates remotely over
+gRPC, in-process syncs the ruleset and evaluates locally -- and they are separate
+suites because they are separately conformant. Any difference between the two
+results is a difference an application would see when it switches resolver,
+which is exactly the class of thing the conformance suite exists to surface.
+
+Everything they share lives here; everything that differs lives in the two
+``test_*_conformance`` modules next to it, where a reader can see the whole of a
+resolver's declaration in one place.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from openfeature.contrib.provider.flagd import FlagdProvider
+from openfeature.contrib.provider.flagd.config import ResolverType
+from openfeature.contrib.tools.provider_tck import (
+    Capability,
+    HttpControl,
+    TckConfig,
+)
+from openfeature.provider import FeatureProvider
+from tests.e2e.flagd_container import FlagdContainer
+
+__all__ = ["ResolverSuite", "build_config"]
+
+# Timings. flagd exposes several and they interact, so they are named here once
+# rather than scattered through two suites.
+
+DEADLINE_MS = 5000
+"""How long a provider blocks during initialisation before giving up.
+
+This, and not ``TckConfig.ready_timeout``, is what actually bounds flagd's
+initialisation: the resolvers block inside ``initialize`` and raise
+``ProviderNotReadyError`` when it expires (grpc.py:175, grpc_watcher.py:151).
+Generous because every scenario is preceded by a control-API ``/start``, which
+restarts the flagd process -- so a provider is routinely built against a backend
+that came up milliseconds ago.
+
+For the RPC resolver it also bounds each individual resolution call.
+"""
+
+RETRY_BACKOFF_MS = 500
+"""Initial delay before a reconnect attempt. Short, so an ended outage is noticed quickly."""
+
+RETRY_BACKOFF_MAX_MS = 5000
+"""Longest delay between reconnect attempts, and the wait between stream retries.
+
+Kept at or above :data:`DEADLINE_MS`: the provider passes the deadline to gRPC as
+``grpc.min_reconnect_backoff_ms`` and this as ``grpc.max_reconnect_backoff_ms``
+(grpc.py:90-92), so a smaller value here would be a channel configured with a
+minimum backoff above its maximum.
+"""
+
+RETRY_GRACE_PERIOD_SECONDS = 30
+"""How long a disconnected provider stays STALE before escalating to ERROR.
+
+Load-bearing for the ``@stale`` scenario. Both resolvers go STALE the moment the
+channel fails and start a timer that escalates to ERROR when this expires
+(grpc.py:204-210, grpc_watcher.py:180-186). Too short a value turns a scenario
+about staleness into one about failure, because the outage lasts as long as the
+scenario needs to observe it.
+"""
+
+STREAM_DEADLINE_MS = 0
+"""Disable periodic stream recycling, as the existing e2e suites do.
+
+A recycle is invisible to the provider contract, but disabling it removes a
+source of background reconnects from a suite whose whole subject is what a
+reconnect looks like.
+"""
+
+UNAVAILABLE_DEADLINE_MS = 500
+UNAVAILABLE_GRACE_PERIOD_SECONDS = 1
+UNAVAILABLE_BACKOFF_MS = 5000
+"""Deliberately impatient settings for the provider that cannot reach a backend.
+
+The ``@unavailable`` scenarios assert that failure is reported *promptly*, so a
+provider taking 30 seconds to give up would pass a test about eventual failure
+while failing the one that matters. The backoff is long for the opposite reason:
+after the error is reported, the next failed reconnect attempt would emit
+another ``PROVIDER_STALE`` and move the provider out of the ERROR state the
+scenario is about to assert.
+"""
+
+EVENT_TIMEOUT = 20.0
+"""Seconds to wait for a provider event.
+
+Comfortably above :data:`RETRY_BACKOFF_MAX_MS`, which is how long a provider may
+wait before the reconnect attempt that produces the ``PROVIDER_READY`` ending
+the ``@stale`` scenario.
+"""
+
+
+@dataclass(frozen=True)
+class ResolverSuite:
+    """What differs between the two resolvers' suites."""
+
+    name: str
+    """Names the suite in test output and scopes its OpenFeature domain."""
+
+    resolver_type: ResolverType
+
+    capabilities: frozenset[Capability]
+    """Derived from reading the resolver's event emission, not from running the suite.
+
+    See each suite module for the evidence behind every entry, and behind every
+    omission.
+    """
+
+    ready_timeout: float
+
+
+def build_config(
+    suite: ResolverSuite,
+    container: FlagdContainer,
+    control: HttpControl,
+    closed_port: int,
+) -> TckConfig:
+    """Wire one resolver up to the running testbed.
+
+    The ports are read here, after the stack is up: the testbed maps host ports
+    dynamically, so they do not exist earlier -- and they stay valid for the
+    whole session because nothing ever restarts a container. Outages are
+    simulated inside the running stack through ``control`` instead.
+    """
+    port = container.get_port(suite.resolver_type)
+
+    def new_provider() -> FeatureProvider:
+        return FlagdProvider(
+            resolver_type=suite.resolver_type,
+            host="localhost",
+            port=port,
+            deadline_ms=DEADLINE_MS,
+            stream_deadline_ms=STREAM_DEADLINE_MS,
+            retry_backoff_ms=RETRY_BACKOFF_MS,
+            retry_backoff_max_ms=RETRY_BACKOFF_MAX_MS,
+            retry_grace_period=RETRY_GRACE_PERIOD_SECONDS,
+        )
+
+    def new_unavailable_provider() -> FeatureProvider:
+        # Pointed at a closed port on localhost, never at the backend under
+        # test: that has to stay up, and simulated outages belong to the control
+        # API.
+        return FlagdProvider(
+            resolver_type=suite.resolver_type,
+            host="localhost",
+            port=closed_port,
+            deadline_ms=UNAVAILABLE_DEADLINE_MS,
+            stream_deadline_ms=STREAM_DEADLINE_MS,
+            retry_backoff_ms=UNAVAILABLE_BACKOFF_MS,
+            retry_backoff_max_ms=UNAVAILABLE_BACKOFF_MS,
+            retry_grace_period=UNAVAILABLE_GRACE_PERIOD_SECONDS,
+        )
+
+    return TckConfig(
+        name=suite.name,
+        control=control,
+        new_provider=new_provider,
+        new_unavailable_provider=new_unavailable_provider,
+        capabilities=suite.capabilities,
+        event_timeout=EVENT_TIMEOUT,
+        ready_timeout=suite.ready_timeout,
+    )

providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py

@@ -0,0 +1,96 @@
+"""The OpenFeature provider conformance suite, run against flagd's in-process resolver.
+
+In-process syncs the whole ruleset over flagd's sync API and evaluates locally,
+so unlike RPC the type-checking, the variant selection and the reason all come
+from ``openfeature-flagd-core`` in this process rather than from the server. Any
+difference in the results is a difference an application would see when it
+switches resolver, which is why this is a separate suite rather than a
+parametrisation of the RPC one.
+"""
+
+from __future__ import annotations
+
+import pytest
+from pytest_bdd import scenarios
+
+from openfeature.contrib.provider.flagd.config import ResolverType
+from openfeature.contrib.tools.provider_tck import (
+    Capability,
+    HttpControl,
+    TckConfig,
+    features_path,
+)
+from tests.e2e.flagd_container import FlagdContainer
+from tests.tck.suite import ResolverSuite, build_config
+
+# Every capability below is declared on the strength of a line of provider code,
+# not on the strength of a green run.
+#
+#   EVENTS
+#     grpc_watcher.py:262 emits PROVIDER_READY once the first sync payload has
+#     been applied -- note "applied", not "received": the ruleset is written to
+#     the evaluator at grpc_watcher.py:254 before ready is emitted, so a scenario
+#     that evaluates immediately after ready cannot race the first sync.
+#
+#   STALE
+#     grpc_watcher.py:178-190: the channel-connectivity callback emits
+#     PROVIDER_STALE on TRANSIENT_FAILURE and starts a timer that escalates to
+#     PROVIDER_ERROR only once retry_grace_period expires.
+#
+#   CONFIGURATION_CHANGE
+#     in_process.py:34 emits PROVIDER_CONFIGURATION_CHANGED naming exactly the
+#     keys that FlagdCore reports as changed, from every sync payload the watcher
+#     applies.
+#
+#   OBJECT
+#     in_process.py:122 resolves structured values from the local ruleset.
+#
+#   UNAVAILABLE_INIT
+#     grpc_watcher.py:151 raises ProviderNotReadyError once the blocking init
+#     deadline passes without a synced ruleset, which the SDK's registry turns
+#     into PROVIDER_ERROR.
+#
+#   STRICT_NUMERIC_TYPING
+#     Local, and strict: flagd_core.py:25 admits only `int` for an integer
+#     request, and flagd_core.py:228-231 raises TypeMismatchError for anything
+#     else -- so `float-flag`'s 0.5 is reported as a mismatch rather than
+#     narrowed to 0. (The float mapping at flagd_core.py:26 is deliberately the
+#     wider one, `(int, float)`, but widening towards float loses nothing.)
+#
+# Not declared, and why:
+#
+#   TARGETING, CACHING
+#     Reserved in the Capability enum; no scenario carries either tag. Declaring
+#     a capability nothing exercises would be a claim with no evidence behind it,
+#     so they are left out of both suites.
+IN_PROCESS_CAPABILITIES = frozenset(
+    {
+        Capability.EVENTS,
+        Capability.STALE,
+        Capability.CONFIGURATION_CHANGE,
+        Capability.OBJECT,
+        Capability.UNAVAILABLE_INIT,
+        Capability.STRICT_NUMERIC_TYPING,
+    }
+)
+
+IN_PROCESS_SUITE = ResolverSuite(
+    name="flagd-in-process",
+    resolver_type=ResolverType.IN_PROCESS,
+    capabilities=IN_PROCESS_CAPABILITIES,
+    # In-process transfers and applies the whole ruleset before reporting ready,
+    # so it needs more headroom than RPC.
+    ready_timeout=60.0,
+)
+
+
+@pytest.fixture(scope="session")
+def tck_config(
+    flagd_testbed: FlagdContainer,
+    flagd_control: HttpControl,
+    closed_port: int,
+) -> TckConfig:
+    return build_config(IN_PROCESS_SUITE, flagd_testbed, flagd_control, closed_port)
+
+
+scenarios(features_path())