test(ofrep): run the provider conformance suite against flagd's OFREP API

providers/openfeature-provider-ofrep/pyproject.toml

@@ -29,12 +29,23 @@ Homepage = "https://github.com/open-feature/python-sdk-contrib"
 dev = [
   "coverage[toml]>=7.10.0,<8.0.0",
   "mypy>=1.18.0,<2.0.0",
+  # The OpenFeature provider conformance suite. Ships the feature files, the flag
+  # set and the control-API client, and registers its step definitions through a
+  # pytest11 entry point, so tests/tck needs no conftest of its own for them.
+  "openfeature-provider-tck",
   "poethepoet>=0.37.0",
   "pytest>=9.0.0,<10.0.0",
+  "pytest-bdd>=8.1.0,<9.0.0",
   "requests-mock>=1.12.0,<2.0.0",
+  # Starts the flagd testbed, which serves the OFREP API on port 8016 alongside
+  # flagd's own protocols. See tests/tck/testbed.py.
+  "testcontainers>=4.12.0,<5.0.0",
   "types-requests>=2.32.0,<3.0.0",
 ]
 
+[tool.uv.sources]
+openfeature-provider-tck = { workspace = true }
+
 [tool.uv.build-backend]
 module-name = "openfeature"
 module-root = "src"

providers/openfeature-provider-ofrep/tests/tck/conftest.py

@@ -0,0 +1,79 @@
+"""Session fixtures for the OFREP conformance suite, and one recorded deviation.
+
+The stack is started once and never restarted, because compose assigns host
+ports dynamically and cannot preserve them across a restart: a restarted backend
+comes back on a different port, silently invalidating a provider already pointed
+at the old one, and the failure reads as a flaky provider rather than a broken
+test. Scenario isolation comes from the control API instead -- see the
+no-container-restart invariant in the TCK's ``control-api.yaml``.
+"""
+
+from __future__ import annotations
+
+import typing
+
+import pytest
+
+from openfeature.contrib.tools.provider_tck import HttpControl
+from tests.tck.settled_control import SettledControl
+from tests.tck.testbed import FlagdTestbed, running_testbed
+
+
+@pytest.fixture(scope="session")
+def flagd_testbed() -> typing.Iterator[FlagdTestbed]:
+    """The testbed stack, up for the whole session."""
+    yield from running_testbed()
+
+
+@pytest.fixture(scope="session")
+def ofrep_control(flagd_testbed: FlagdTestbed) -> SettledControl:
+    """The control API client, pointed at the testbed's launchpad.
+
+    The launchpad registers only ``/start``, ``/restart``, ``/stop`` and
+    ``/change`` (flagd-testbed ``launchpad/main.go:29-32``), so ``/reset``
+    answers 404 and every ``prepare_scenario`` takes the documented ``/start``
+    fallback. The probe costs one 404 for the whole session.
+
+    Wrapped in :class:`SettledControl` because ``/start`` returns before the
+    backend serves the flag set, and a stateless provider has no initialisation
+    to hide that window behind. See that module -- it is a finding about the
+    control API's guarantee, not a convenience.
+    """
+    return SettledControl(
+        HttpControl(flagd_testbed.get_launchpad_url()),
+        flagd_testbed.get_ofrep_url(),
+    )
+
+
+# ---------------------------------------------------------------------------
+# One known deviation, recorded rather than hidden.
+#
+# A conformance suite that quietly goes green on a scenario it ran and failed is
+# as bad as one that goes green on a scenario it skipped. So the single scenario
+# this provider cannot satisfy is marked xfail(strict=True), which keeps it in
+# the report with its reason attached and fails the suite the moment it starts
+# passing -- so the marker is removed when the bug is fixed rather than
+# lingering as a lie. Same mechanism, and same bug, as the TCK's own self-test
+# (tools/openfeature-provider-tck/tests/conftest.py).
+
+_BOOL_AS_INT = (
+    "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]"
+)
+
+_REASON = (
+    "bool satisfies an Integer request. OFREP is an untyped protocol -- the "
+    "backend returns the JSON value with no knowledge of the requested type -- so "
+    "the whole type check is the provider's, at ofrep/__init__.py:244-256: "
+    "FlagType.INTEGER maps to `int` and the check is isinstance(value, int), which "
+    "bool is a subclass of in Python. boolean-flag requested as an Integer "
+    "therefore returns True with reason STATIC and no error code, where the "
+    "specification requires the code default and TYPE_MISMATCH. The Python SDK "
+    "client type-checks the same way, so fixing only one of the two is not enough. "
+    "See https://github.com/open-feature/python-sdk/issues/619"
+)
+
+
+def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
+    for item in items:
+        if item.name == _BOOL_AS_INT:
+            item.add_marker(pytest.mark.xfail(reason=_REASON, strict=True))

providers/openfeature-provider-ofrep/tests/tck/settled_control.py

@@ -0,0 +1,136 @@
+"""``HttpControl``, plus a wait for the backend to actually serve the flag set.
+
+**The problem this exists for is worth stating carefully, because it is a
+finding rather than a workaround.**
+
+``POST /start`` is specified to reseed flag state to the named configuration's
+baseline (normative requirement 2 in the TCK's ``control-api.yaml``). It is not
+specified to *return only once that state is being served*, and flagd-testbed's
+launchpad does not: it returns as soon as flagd answers ``/readyz``
+(``launchpad/pkg/flagd.go``), which flagd does before its file sources have been
+loaded into the flag store. Measured against this testbed, the window is short
+-- around 40ms -- but it is real and reliably hit:
+
+    start:200 {"errorCode":"FLAG_NOT_FOUND","errorDetails":"flag `float-flag` does not exist"}
+    start:200 {"value":0.5,"key":"float-flag","reason":"STATIC","variant":"half"}
+    start:200 {"errorCode":"FLAG_NOT_FOUND","errorDetails":"flag `float-flag` does not exist"}
+
+The flagd suites never see it, and that is the interesting part. Both flagd
+resolvers block inside ``initialize`` until the evaluation stream is up or the
+ruleset has synced, so their initialisation absorbs the window before any
+scenario evaluates. OFREP is stateless -- no ``initialize``, no connection, no
+warm-up -- so its first evaluation lands directly in the gap and the suite
+reports FLAG_NOT_FOUND for every flag, which reads as a catastrophically broken
+provider.
+
+**A stateless provider is the first adopter with no initialisation to hide a
+backend's warm-up behind**, which makes it the one that discovers whether the
+control API's guarantee is strong enough. It is not: "reseeded" and "serving"
+need to be the same instant, or every stateless provider reimplements this. That
+belongs in the control API contract, and until it is there it belongs here.
+
+**Why this is not cheating.** It manipulates nothing. It is a readiness probe
+over the same public OFREP endpoint the provider uses, on a canonical flag,
+asserting only that the backend has finished doing what ``/start`` already
+promised. No scenario is weakened, no step is bypassed, and no side channel into
+the backend is opened -- the normative control path is still ``HttpControl``,
+which this delegates to unchanged.
+
+Deliberately not a :class:`ConnectionControl`: it has no ``disconnect`` or
+``reconnect``, matching a suite that declares neither ``STALE`` nor
+``UNAVAILABLE_INIT``. The two omissions keep each other honest.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+import urllib.error
+import urllib.request
+
+from openfeature.contrib.tools.provider_tck import HttpControl
+
+__all__ = ["SettledControl"]
+
+PROBE_FLAG_KEY = "boolean-flag"
+"""A canonical flag, used only to ask whether the flag set is being served yet."""
+
+SETTLE_TIMEOUT_SECONDS = 15.0
+"""How long to wait for the backend to serve the flag set after ``/start``.
+
+Two orders of magnitude above the ~40ms observed, because the cost of being
+generous is nothing -- the loop exits on the first success -- while the cost of
+being tight is a suite that fails intermittently on a loaded CI runner and gets
+diagnosed as a provider bug.
+"""
+
+SETTLE_POLL_SECONDS = 0.02
+
+
+class SettledControl:
+    """Delegates to :class:`HttpControl`, then waits for the flags to appear."""
+
+    def __init__(
+        self,
+        control: HttpControl,
+        ofrep_url: str,
+        *,
+        timeout: float = SETTLE_TIMEOUT_SECONDS,
+    ) -> None:
+        self._control = control
+        self._probe_url = (
+            f"{ofrep_url.rstrip('/')}/ofrep/v1/evaluate/flags/{PROBE_FLAG_KEY}"
+        )
+        self._timeout = timeout
+
+    @property
+    def description(self) -> str:
+        return f"{self._control.description}, awaited through the OFREP endpoint"
+
+    def prepare_scenario(self) -> None:
+        self._control.prepare_scenario()
+        self._await_flags()
+
+    def change_flag(self) -> None:
+        self._control.change_flag()
+
+    def _await_flags(self) -> None:
+        """Block until the probe flag resolves, or fail saying what was seen.
+
+        Raising rather than proceeding is deliberate. A scenario allowed to run
+        against a backend that is not serving its flag set does not report a
+        harness problem; it reports FLAG_NOT_FOUND as a conformance result,
+        which is the one outcome a conformance suite must never produce.
+        """
+        deadline = time.monotonic() + self._timeout
+        last = "no response"
+
+        while time.monotonic() < deadline:
+            status, body = self._probe()
+            if status == 200:
+                return
+            last = f"HTTP {status}: {body}"
+            time.sleep(SETTLE_POLL_SECONDS)
+
+        msg = (
+            f"the backend did not serve {PROBE_FLAG_KEY!r} within {self._timeout}s of "
+            f"a successful control-API reseed. Last response from {self._probe_url}: "
+            f"{last}. This is a problem with the stack under test or its control API, "
+            f"not with the provider"
+        )
+        raise RuntimeError(msg)
+
+    def _probe(self) -> tuple[int, str]:
+        request = urllib.request.Request(  # noqa: S310
+            self._probe_url,
+            data=json.dumps({}).encode("utf-8"),
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=5.0) as response:  # noqa: S310
+                return int(response.status), ""
+        except urllib.error.HTTPError as err:
+            return int(err.code), err.read().decode("utf-8", "replace")[:200]
+        except (urllib.error.URLError, OSError) as err:
+            return 0, str(err)

providers/openfeature-provider-ofrep/tests/tck/test_ofrep_conformance.py

@@ -0,0 +1,153 @@
+"""The OpenFeature provider conformance suite, run against the OFREP provider.
+
+OFREP is the vendor-neutral remote evaluation protocol, so what is under test
+here is a pure mapping: one HTTP request per evaluation, and the translation of
+its JSON response -- or its error status -- into typed resolution details. There
+is no cache, no stream and no local ruleset, so unlike the flagd suites there is
+nothing here that a lifecycle could be wrong about.
+
+The backend is flagd, which serves OFREP on port 8016 alongside its own
+protocols, driven through the same launchpad control API and seeded with the
+same canonical flag set as the flagd conformance suites. Running two providers
+against one backend is the point of a cross-provider conformance suite: a
+difference in the results is a difference an application would see when it
+switches provider.
+"""
+
+from __future__ import annotations
+
+import pytest
+from pytest_bdd import scenarios
+
+from openfeature.contrib.provider.ofrep import OFREPProvider
+from openfeature.contrib.tools.provider_tck import (
+    Capability,
+    TckConfig,
+    features_path,
+)
+from openfeature.provider import FeatureProvider
+from tests.tck.settled_control import SettledControl
+from tests.tck.testbed import FlagdTestbed
+
+TIMEOUT_SECONDS = 10.0
+"""Bounds a single OFREP request.
+
+Generous, because every scenario is preceded by a control-API ``/start`` that
+restarts the flagd process, so the first evaluation of a scenario routinely hits
+a backend that came up milliseconds ago. It is the only timing knob this
+provider has (``ofrep/__init__.py:52``); everything else the TCK offers -- event
+timeouts, ready timeouts -- has nothing to bound, for the reasons below.
+"""
+
+# Every capability below is declared on the strength of a line of provider code,
+# not on the strength of a green run.
+#
+#   OBJECT
+#     ofrep/__init__.py:105-113 resolves structured values, and the type check at
+#     ofrep/__init__.py:248 admits `(dict, list)` for FlagType.OBJECT -- so a JSON
+#     object comes back as one rather than being rejected or flattened.
+#
+#   STRICT_NUMERIC_TYPING
+#     ofrep/__init__.py:250 maps FlagType.INTEGER to `int`, and the isinstance
+#     check at ofrep/__init__.py:255 fails for the float 0.5, raising
+#     TypeMismatchError. So float-flag's 0.5 is reported as a mismatch rather
+#     than narrowed to 0. Worth stating plainly that this is the provider's own
+#     doing: OFREP is untyped on the wire, the request carries no type at all,
+#     and flagd returns 0.5 whatever was asked for -- so unlike flagd-RPC, where
+#     the server answers INVALID_ARGUMENT, there is no backend here to catch a
+#     numeric mismatch. Every type decision in this suite is made at those two
+#     lines, which is also why the one deviation recorded in conftest.py lives
+#     there.
+#
+# Not declared, and why. Each is a fact about the provider, established by
+# reading it -- OFREPProvider is stateless: it holds a requests.Session and a
+# rate-limit timestamp, and nothing else survives between evaluations.
+#
+#   LIFECYCLE
+#     Not in the Capability enum on this branch yet (it lands with
+#     feat/provider-tck), and it would not be declared once it does.
+#     OFREPProvider does not override `initialize`, so it inherits
+#     AbstractProvider's, which is `pass` (python-sdk
+#     openfeature/provider/__init__.py:138-139). Nothing contacts the backend
+#     before the first evaluation, so initialisation has no outcome to observe.
+#     lifecycle.feature is gated on @events at feature level on this branch and
+#     skips for that reason; when it is retagged to @lifecycle it must keep
+#     skipping, for this one.
+#
+#   EVENTS
+#     The provider never emits. It extends AbstractProvider, so it inherits
+#     `attach`, but `_on_emit` is never called anywhere in
+#     ofrep/__init__.py -- there is no stream, no poll and no background thread
+#     to notice anything worth emitting about.
+#
+#     The SDK's registry does dispatch PROVIDER_READY around `initialize` for any
+#     provider (python-sdk openfeature/provider/_registry.py:73-77), so declaring
+#     EVENTS would make lifecycle.feature's readiness scenario pass without
+#     demonstrating anything -- a NoOpProvider passes it identically. That is
+#     exactly the vacuity the @lifecycle capability was split out to end, and
+#     claiming the capability to collect the pass would be the dishonest use of
+#     it.
+#
+#   STALE, CONFIGURATION_CHANGE
+#     Both are event capabilities and follow from EVENTS. There is no connection
+#     to lose -- every evaluation is an independent HTTP request -- so there is no
+#     state between them that could go stale, and nothing watches the backend for
+#     a configuration change. events.feature is gated @events at feature level
+#     and skips as a whole.
+#
+#     Note that a *change* is nonetheless visible to an application: the next
+#     evaluation issues a fresh request and returns the new value. What is
+#     missing is the signal, and the @configuration-change scenario asserts the
+#     event as well as the behaviour, deliberately -- a provider that changes
+#     silently is not conformant, it is just not broken.
+#
+#   UNAVAILABLE_INIT
+#     A provider pointed at a closed port reaches READY, because `initialize`
+#     does nothing and the registry dispatches PROVIDER_READY unconditionally
+#     (python-sdk openfeature/provider/_registry.py:73-77). The failure surfaces
+#     on the first evaluation as GeneralError from ofrep/__init__.py:167, not as
+#     PROVIDER_ERROR, so the scenario's premise does not hold. `TckConfig` also
+#     rejects the capability without a `new_unavailable_provider`, and none is
+#     supplied here for the same reason.
+#
+#   TARGETING, CACHING
+#     Reserved in the Capability enum; no scenario carries either tag. Declaring
+#     a capability nothing exercises would be a claim with no evidence behind it.
+#
+# The result matches the Go and Java OFREP adoptions, which reached the same two
+# capabilities from the same architecture, independently.
+CAPABILITIES = frozenset(
+    {
+        Capability.OBJECT,
+        Capability.STRICT_NUMERIC_TYPING,
+    }
+)
+
+
+@pytest.fixture(scope="session")
+def tck_config(
+    flagd_testbed: FlagdTestbed,
+    ofrep_control: SettledControl,
+) -> TckConfig:
+    """Wire the provider up to the running testbed.
+
+    The port is read here, after the stack is up: compose maps host ports
+    dynamically, so it does not exist earlier -- and it stays valid for the whole
+    session because nothing ever restarts a container. Outages, had this suite
+    any use for them, would be simulated inside the running stack through
+    ``ofrep_control``.
+    """
+    base_url = flagd_testbed.get_ofrep_url()
+
+    def new_provider() -> FeatureProvider:
+        return OFREPProvider(base_url, timeout=TIMEOUT_SECONDS)
+
+    return TckConfig(
+        name="ofrep",
+        control=ofrep_control,
+        new_provider=new_provider,
+        capabilities=CAPABILITIES,
+    )
+
+
+scenarios(features_path())