From 30b3439967f772234aba1d1116be7a3a1715becd Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 15:15:56 +0200 Subject: [PATCH] test(ofrep): run the provider conformance suite against flagd's OFREP API Adopts the OpenFeature provider conformance suite in the OFREP provider. No new infrastructure. flagd serves the OFREP API on port 8016 alongside its own protocols, and flagd-testbed's compose file already publishes it, so the OFREP provider runs against the existing testbed, seeded with the same canonical flag set, driven through the same launchpad control API as the flagd suites. Running two providers against one backend is the point of a cross-provider conformance suite: a difference in the results is a difference an application would see when it switches provider. tests/e2e/flagd_container.FlagdContainer would have been the natural thing to reuse and is not importable here -- a package's tests are not part of its distribution -- so tests/tck/testbed.py drives compose directly. What it duplicates is deliberately minimal: compose up, read two mapped ports, poll /readyz. This is a concrete instance of the "no shared containerised-backend helper" gap the TCK's README records. Two capabilities, both on the strength of a line of provider code rather than of a green run: OBJECT and STRICT_NUMERIC_TYPING. The same two the Go and Java OFREP adoptions reached independently, from the same architecture. Every omission is a fact about the provider. OFREPProvider is stateless -- it holds a requests.Session and a rate-limit timestamp, and nothing else survives between evaluations. It does not override initialize, so it inherits AbstractProvider's, which is `pass`, and it never emits: `_on_emit` is not called anywhere in the provider. So EVENTS, STALE and CONFIGURATION_CHANGE have nothing behind them, and UNAVAILABLE_INIT is false in the strong sense -- a provider pointed at a closed port reaches READY, because the SDK's registry dispatches PROVIDER_READY around an initialize that does nothing. events.feature and lifecycle.feature are gated at feature level and skip with their reasons; 24 of the 29 scenarios run. @lifecycle, which lands on the TCK branch this is stacked under, would also be withheld once it is available here: nothing contacts the backend before the first evaluation, so initialisation has no outcome to observe. That capability was split out of EVENTS precisely so a stateless provider can decline it accurately, and this is the case it was split out for. One scenario is marked xfail(strict=True): boolean-flag requested as an Integer. OFREP is untyped on the wire -- the request carries no type and the backend returns the JSON value regardless -- so the whole type check is the provider's, and it is isinstance(value, int), which bool is a subclass of in Python. The value True comes back with reason STATIC and no error code where the specification requires the code default and TYPE_MISMATCH. The SDK client type-checks the same way, so this is the provider-side half of open-feature/python-sdk#619 and fixing one half is not enough. Strict, so the marker fails the suite once it starts passing rather than lingering as a lie. Recorded as a finding: POST /start returns before the backend serves the flag set. The control API specifies that /start reseeds flag state; it does not specify that it returns only once that state is being served, and flagd-testbed's launchpad returns as soon as flagd answers /readyz, which is roughly 40ms before its file sources are in the flag store. The flagd suites never see this because both resolvers block inside initialize until the stream is up or the ruleset has synced, absorbing the window. A stateless provider is the first adopter with no initialisation to hide a backend's warm-up behind, and its first evaluation lands squarely in the gap -- reported, before the fix, as FLAG_NOT_FOUND on every flag. SettledControl closes it by delegating to HttpControl and then polling the public OFREP endpoint until the flag set is actually served. It manipulates nothing and weakens no scenario, but "reseeded" and "serving" should be the same instant in the control API contract, and until they are this belongs in the adoption. 23 passed, 5 skipped, 1 xfailed. Signed-off-by: Simon Schrottner --- .../openfeature-provider-ofrep/pyproject.toml | 11 ++ .../tests/tck/__init__.py | 0 .../tests/tck/conftest.py | 79 ++++++++ .../tests/tck/settled_control.py | 136 ++++++++++++++ .../tests/tck/test_ofrep_conformance.py | 153 ++++++++++++++++ .../tests/tck/testbed.py | 172 ++++++++++++++++++ uv.lock | 6 + 7 files changed, 557 insertions(+) create mode 100644 providers/openfeature-provider-ofrep/tests/tck/__init__.py create mode 100644 providers/openfeature-provider-ofrep/tests/tck/conftest.py create mode 100644 providers/openfeature-provider-ofrep/tests/tck/settled_control.py create mode 100644 providers/openfeature-provider-ofrep/tests/tck/test_ofrep_conformance.py create mode 100644 providers/openfeature-provider-ofrep/tests/tck/testbed.py diff --git a/providers/openfeature-provider-ofrep/pyproject.toml b/providers/openfeature-provider-ofrep/pyproject.toml index bb054ad5..fbe7a04b 100644 --- a/providers/openfeature-provider-ofrep/pyproject.toml +++ b/providers/openfeature-provider-ofrep/pyproject.toml @@ -29,12 +29,23 @@ Homepage = "https://github.com/open-feature/python-sdk-contrib" dev = [ "coverage[toml]>=7.10.0,<8.0.0", "mypy>=1.18.0,<2.0.0", + # The OpenFeature provider conformance suite. Ships the feature files, the flag + # set and the control-API client, and registers its step definitions through a + # pytest11 entry point, so tests/tck needs no conftest of its own for them. + "openfeature-provider-tck", "poethepoet>=0.37.0", "pytest>=9.0.0,<10.0.0", + "pytest-bdd>=8.1.0,<9.0.0", "requests-mock>=1.12.0,<2.0.0", + # Starts the flagd testbed, which serves the OFREP API on port 8016 alongside + # flagd's own protocols. See tests/tck/testbed.py. + "testcontainers>=4.12.0,<5.0.0", "types-requests>=2.32.0,<3.0.0", ] +[tool.uv.sources] +openfeature-provider-tck = { workspace = true } + [tool.uv.build-backend] module-name = "openfeature" module-root = "src" diff --git a/providers/openfeature-provider-ofrep/tests/tck/__init__.py b/providers/openfeature-provider-ofrep/tests/tck/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/openfeature-provider-ofrep/tests/tck/conftest.py b/providers/openfeature-provider-ofrep/tests/tck/conftest.py new file mode 100644 index 00000000..c692ab0d --- /dev/null +++ b/providers/openfeature-provider-ofrep/tests/tck/conftest.py @@ -0,0 +1,79 @@ +"""Session fixtures for the OFREP conformance suite, and one recorded deviation. + +The stack is started once and never restarted, because compose assigns host +ports dynamically and cannot preserve them across a restart: a restarted backend +comes back on a different port, silently invalidating a provider already pointed +at the old one, and the failure reads as a flaky provider rather than a broken +test. Scenario isolation comes from the control API instead -- see the +no-container-restart invariant in the TCK's ``control-api.yaml``. +""" + +from __future__ import annotations + +import typing + +import pytest + +from openfeature.contrib.tools.provider_tck import HttpControl +from tests.tck.settled_control import SettledControl +from tests.tck.testbed import FlagdTestbed, running_testbed + + +@pytest.fixture(scope="session") +def flagd_testbed() -> typing.Iterator[FlagdTestbed]: + """The testbed stack, up for the whole session.""" + yield from running_testbed() + + +@pytest.fixture(scope="session") +def ofrep_control(flagd_testbed: FlagdTestbed) -> SettledControl: + """The control API client, pointed at the testbed's launchpad. + + The launchpad registers only ``/start``, ``/restart``, ``/stop`` and + ``/change`` (flagd-testbed ``launchpad/main.go:29-32``), so ``/reset`` + answers 404 and every ``prepare_scenario`` takes the documented ``/start`` + fallback. The probe costs one 404 for the whole session. + + Wrapped in :class:`SettledControl` because ``/start`` returns before the + backend serves the flag set, and a stateless provider has no initialisation + to hide that window behind. See that module -- it is a finding about the + control API's guarantee, not a convenience. + """ + return SettledControl( + HttpControl(flagd_testbed.get_launchpad_url()), + flagd_testbed.get_ofrep_url(), + ) + + +# --------------------------------------------------------------------------- +# One known deviation, recorded rather than hidden. +# +# A conformance suite that quietly goes green on a scenario it ran and failed is +# as bad as one that goes green on a scenario it skipped. So the single scenario +# this provider cannot satisfy is marked xfail(strict=True), which keeps it in +# the report with its reason attached and fails the suite the moment it starts +# passing -- so the marker is removed when the bug is fixed rather than +# lingering as a lie. Same mechanism, and same bug, as the TCK's own self-test +# (tools/openfeature-provider-tck/tests/conftest.py). + +_BOOL_AS_INT = ( + "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" +) + +_REASON = ( + "bool satisfies an Integer request. OFREP is an untyped protocol -- the " + "backend returns the JSON value with no knowledge of the requested type -- so " + "the whole type check is the provider's, at ofrep/__init__.py:244-256: " + "FlagType.INTEGER maps to `int` and the check is isinstance(value, int), which " + "bool is a subclass of in Python. boolean-flag requested as an Integer " + "therefore returns True with reason STATIC and no error code, where the " + "specification requires the code default and TYPE_MISMATCH. The Python SDK " + "client type-checks the same way, so fixing only one of the two is not enough. " + "See https://github.com/open-feature/python-sdk/issues/619" +) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + for item in items: + if item.name == _BOOL_AS_INT: + item.add_marker(pytest.mark.xfail(reason=_REASON, strict=True)) diff --git a/providers/openfeature-provider-ofrep/tests/tck/settled_control.py b/providers/openfeature-provider-ofrep/tests/tck/settled_control.py new file mode 100644 index 00000000..e834b47b --- /dev/null +++ b/providers/openfeature-provider-ofrep/tests/tck/settled_control.py @@ -0,0 +1,136 @@ +"""``HttpControl``, plus a wait for the backend to actually serve the flag set. + +**The problem this exists for is worth stating carefully, because it is a +finding rather than a workaround.** + +``POST /start`` is specified to reseed flag state to the named configuration's +baseline (normative requirement 2 in the TCK's ``control-api.yaml``). It is not +specified to *return only once that state is being served*, and flagd-testbed's +launchpad does not: it returns as soon as flagd answers ``/readyz`` +(``launchpad/pkg/flagd.go``), which flagd does before its file sources have been +loaded into the flag store. Measured against this testbed, the window is short +-- around 40ms -- but it is real and reliably hit: + + start:200 {"errorCode":"FLAG_NOT_FOUND","errorDetails":"flag `float-flag` does not exist"} + start:200 {"value":0.5,"key":"float-flag","reason":"STATIC","variant":"half"} + start:200 {"errorCode":"FLAG_NOT_FOUND","errorDetails":"flag `float-flag` does not exist"} + +The flagd suites never see it, and that is the interesting part. Both flagd +resolvers block inside ``initialize`` until the evaluation stream is up or the +ruleset has synced, so their initialisation absorbs the window before any +scenario evaluates. OFREP is stateless -- no ``initialize``, no connection, no +warm-up -- so its first evaluation lands directly in the gap and the suite +reports FLAG_NOT_FOUND for every flag, which reads as a catastrophically broken +provider. + +**A stateless provider is the first adopter with no initialisation to hide a +backend's warm-up behind**, which makes it the one that discovers whether the +control API's guarantee is strong enough. It is not: "reseeded" and "serving" +need to be the same instant, or every stateless provider reimplements this. That +belongs in the control API contract, and until it is there it belongs here. + +**Why this is not cheating.** It manipulates nothing. It is a readiness probe +over the same public OFREP endpoint the provider uses, on a canonical flag, +asserting only that the backend has finished doing what ``/start`` already +promised. No scenario is weakened, no step is bypassed, and no side channel into +the backend is opened -- the normative control path is still ``HttpControl``, +which this delegates to unchanged. + +Deliberately not a :class:`ConnectionControl`: it has no ``disconnect`` or +``reconnect``, matching a suite that declares neither ``STALE`` nor +``UNAVAILABLE_INIT``. The two omissions keep each other honest. +""" + +from __future__ import annotations + +import json +import time +import urllib.error +import urllib.request + +from openfeature.contrib.tools.provider_tck import HttpControl + +__all__ = ["SettledControl"] + +PROBE_FLAG_KEY = "boolean-flag" +"""A canonical flag, used only to ask whether the flag set is being served yet.""" + +SETTLE_TIMEOUT_SECONDS = 15.0 +"""How long to wait for the backend to serve the flag set after ``/start``. + +Two orders of magnitude above the ~40ms observed, because the cost of being +generous is nothing -- the loop exits on the first success -- while the cost of +being tight is a suite that fails intermittently on a loaded CI runner and gets +diagnosed as a provider bug. +""" + +SETTLE_POLL_SECONDS = 0.02 + + +class SettledControl: + """Delegates to :class:`HttpControl`, then waits for the flags to appear.""" + + def __init__( + self, + control: HttpControl, + ofrep_url: str, + *, + timeout: float = SETTLE_TIMEOUT_SECONDS, + ) -> None: + self._control = control + self._probe_url = ( + f"{ofrep_url.rstrip('/')}/ofrep/v1/evaluate/flags/{PROBE_FLAG_KEY}" + ) + self._timeout = timeout + + @property + def description(self) -> str: + return f"{self._control.description}, awaited through the OFREP endpoint" + + def prepare_scenario(self) -> None: + self._control.prepare_scenario() + self._await_flags() + + def change_flag(self) -> None: + self._control.change_flag() + + def _await_flags(self) -> None: + """Block until the probe flag resolves, or fail saying what was seen. + + Raising rather than proceeding is deliberate. A scenario allowed to run + against a backend that is not serving its flag set does not report a + harness problem; it reports FLAG_NOT_FOUND as a conformance result, + which is the one outcome a conformance suite must never produce. + """ + deadline = time.monotonic() + self._timeout + last = "no response" + + while time.monotonic() < deadline: + status, body = self._probe() + if status == 200: + return + last = f"HTTP {status}: {body}" + time.sleep(SETTLE_POLL_SECONDS) + + msg = ( + f"the backend did not serve {PROBE_FLAG_KEY!r} within {self._timeout}s of " + f"a successful control-API reseed. Last response from {self._probe_url}: " + f"{last}. This is a problem with the stack under test or its control API, " + f"not with the provider" + ) + raise RuntimeError(msg) + + def _probe(self) -> tuple[int, str]: + request = urllib.request.Request( # noqa: S310 + self._probe_url, + data=json.dumps({}).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=5.0) as response: # noqa: S310 + return int(response.status), "" + except urllib.error.HTTPError as err: + return int(err.code), err.read().decode("utf-8", "replace")[:200] + except (urllib.error.URLError, OSError) as err: + return 0, str(err) diff --git a/providers/openfeature-provider-ofrep/tests/tck/test_ofrep_conformance.py b/providers/openfeature-provider-ofrep/tests/tck/test_ofrep_conformance.py new file mode 100644 index 00000000..9f9b47c1 --- /dev/null +++ b/providers/openfeature-provider-ofrep/tests/tck/test_ofrep_conformance.py @@ -0,0 +1,153 @@ +"""The OpenFeature provider conformance suite, run against the OFREP provider. + +OFREP is the vendor-neutral remote evaluation protocol, so what is under test +here is a pure mapping: one HTTP request per evaluation, and the translation of +its JSON response -- or its error status -- into typed resolution details. There +is no cache, no stream and no local ruleset, so unlike the flagd suites there is +nothing here that a lifecycle could be wrong about. + +The backend is flagd, which serves OFREP on port 8016 alongside its own +protocols, driven through the same launchpad control API and seeded with the +same canonical flag set as the flagd conformance suites. Running two providers +against one backend is the point of a cross-provider conformance suite: a +difference in the results is a difference an application would see when it +switches provider. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.provider.ofrep import OFREPProvider +from openfeature.contrib.tools.provider_tck import ( + Capability, + TckConfig, + features_path, +) +from openfeature.provider import FeatureProvider +from tests.tck.settled_control import SettledControl +from tests.tck.testbed import FlagdTestbed + +TIMEOUT_SECONDS = 10.0 +"""Bounds a single OFREP request. + +Generous, because every scenario is preceded by a control-API ``/start`` that +restarts the flagd process, so the first evaluation of a scenario routinely hits +a backend that came up milliseconds ago. It is the only timing knob this +provider has (``ofrep/__init__.py:52``); everything else the TCK offers -- event +timeouts, ready timeouts -- has nothing to bound, for the reasons below. +""" + +# Every capability below is declared on the strength of a line of provider code, +# not on the strength of a green run. +# +# OBJECT +# ofrep/__init__.py:105-113 resolves structured values, and the type check at +# ofrep/__init__.py:248 admits `(dict, list)` for FlagType.OBJECT -- so a JSON +# object comes back as one rather than being rejected or flattened. +# +# STRICT_NUMERIC_TYPING +# ofrep/__init__.py:250 maps FlagType.INTEGER to `int`, and the isinstance +# check at ofrep/__init__.py:255 fails for the float 0.5, raising +# TypeMismatchError. So float-flag's 0.5 is reported as a mismatch rather +# than narrowed to 0. Worth stating plainly that this is the provider's own +# doing: OFREP is untyped on the wire, the request carries no type at all, +# and flagd returns 0.5 whatever was asked for -- so unlike flagd-RPC, where +# the server answers INVALID_ARGUMENT, there is no backend here to catch a +# numeric mismatch. Every type decision in this suite is made at those two +# lines, which is also why the one deviation recorded in conftest.py lives +# there. +# +# Not declared, and why. Each is a fact about the provider, established by +# reading it -- OFREPProvider is stateless: it holds a requests.Session and a +# rate-limit timestamp, and nothing else survives between evaluations. +# +# LIFECYCLE +# Not in the Capability enum on this branch yet (it lands with +# feat/provider-tck), and it would not be declared once it does. +# OFREPProvider does not override `initialize`, so it inherits +# AbstractProvider's, which is `pass` (python-sdk +# openfeature/provider/__init__.py:138-139). Nothing contacts the backend +# before the first evaluation, so initialisation has no outcome to observe. +# lifecycle.feature is gated on @events at feature level on this branch and +# skips for that reason; when it is retagged to @lifecycle it must keep +# skipping, for this one. +# +# EVENTS +# The provider never emits. It extends AbstractProvider, so it inherits +# `attach`, but `_on_emit` is never called anywhere in +# ofrep/__init__.py -- there is no stream, no poll and no background thread +# to notice anything worth emitting about. +# +# The SDK's registry does dispatch PROVIDER_READY around `initialize` for any +# provider (python-sdk openfeature/provider/_registry.py:73-77), so declaring +# EVENTS would make lifecycle.feature's readiness scenario pass without +# demonstrating anything -- a NoOpProvider passes it identically. That is +# exactly the vacuity the @lifecycle capability was split out to end, and +# claiming the capability to collect the pass would be the dishonest use of +# it. +# +# STALE, CONFIGURATION_CHANGE +# Both are event capabilities and follow from EVENTS. There is no connection +# to lose -- every evaluation is an independent HTTP request -- so there is no +# state between them that could go stale, and nothing watches the backend for +# a configuration change. events.feature is gated @events at feature level +# and skips as a whole. +# +# Note that a *change* is nonetheless visible to an application: the next +# evaluation issues a fresh request and returns the new value. What is +# missing is the signal, and the @configuration-change scenario asserts the +# event as well as the behaviour, deliberately -- a provider that changes +# silently is not conformant, it is just not broken. +# +# UNAVAILABLE_INIT +# A provider pointed at a closed port reaches READY, because `initialize` +# does nothing and the registry dispatches PROVIDER_READY unconditionally +# (python-sdk openfeature/provider/_registry.py:73-77). The failure surfaces +# on the first evaluation as GeneralError from ofrep/__init__.py:167, not as +# PROVIDER_ERROR, so the scenario's premise does not hold. `TckConfig` also +# rejects the capability without a `new_unavailable_provider`, and none is +# supplied here for the same reason. +# +# TARGETING, CACHING +# Reserved in the Capability enum; no scenario carries either tag. Declaring +# a capability nothing exercises would be a claim with no evidence behind it. +# +# The result matches the Go and Java OFREP adoptions, which reached the same two +# capabilities from the same architecture, independently. +CAPABILITIES = frozenset( + { + Capability.OBJECT, + Capability.STRICT_NUMERIC_TYPING, + } +) + + +@pytest.fixture(scope="session") +def tck_config( + flagd_testbed: FlagdTestbed, + ofrep_control: SettledControl, +) -> TckConfig: + """Wire the provider up to the running testbed. + + The port is read here, after the stack is up: compose maps host ports + dynamically, so it does not exist earlier -- and it stays valid for the whole + session because nothing ever restarts a container. Outages, had this suite + any use for them, would be simulated inside the running stack through + ``ofrep_control``. + """ + base_url = flagd_testbed.get_ofrep_url() + + def new_provider() -> FeatureProvider: + return OFREPProvider(base_url, timeout=TIMEOUT_SECONDS) + + return TckConfig( + name="ofrep", + control=ofrep_control, + new_provider=new_provider, + capabilities=CAPABILITIES, + ) + + +scenarios(features_path()) diff --git a/providers/openfeature-provider-ofrep/tests/tck/testbed.py b/providers/openfeature-provider-ofrep/tests/tck/testbed.py new file mode 100644 index 00000000..feb6930f --- /dev/null +++ b/providers/openfeature-provider-ofrep/tests/tck/testbed.py @@ -0,0 +1,172 @@ +"""The flagd testbed, started for the OFREP conformance suite. + +flagd serves the **OFREP** HTTP API alongside its own protocols -- port 8016, +next to 8013 for RPC and 8015 for sync -- and ``flagd-testbed``'s compose file +already publishes it. So an OFREP provider needs no backend of its own: it runs +against the same stack, seeded with the same canonical flag set, driven through +the same launchpad control API as the flagd suites. A second stack would be a +second definition of "the canonical flags", which is the one thing a conformance +suite exists to prevent. + +**Why this is not ``tests/e2e/flagd_container.FlagdContainer``.** That helper +would be the natural thing to reuse, and it is not importable here: it lives +under ``providers/openfeature-provider-flagd/tests/``, and a package's tests are +not part of its distribution, so nothing in this package can import it. It also +depends on ``grpcio`` and ``grpcio-health-checking`` for its readiness probe, +neither of which the OFREP provider has any other reason to pull in. What is +duplicated is therefore deliberately minimal -- compose up, read two mapped +ports, poll one HTTP endpoint -- and this is a concrete instance of the "no +shared containerised-backend helper" gap the TCK's README already records. + +The compose file is reached through the flagd package's ``test-harness`` +submodule rather than a second checkout of the same repository. Run +``git submodule update --init`` if it is missing; :func:`testbed_path` says so +rather than failing inside testcontainers. +""" + +from __future__ import annotations + +import os +import tempfile +import time +import typing +import urllib.error +import urllib.request +from pathlib import Path + +from testcontainers.compose import DockerCompose + +__all__ = ["LAUNCHPAD_PORT", "OFREP_PORT", "FlagdTestbed", "testbed_path"] + +OFREP_PORT = 8016 +"""flagd's OFREP HTTP port. + +flagd's own default (``flags.Int32P("ofrep-port", "r", 8016, ...)`` in flagd's +``cmd/start.go``), published unchanged by the testbed's compose file. The +testbed's launchpad starts flagd with no ``--ofrep-port`` override, so this is +what it listens on. +""" + +HEALTH_PORT = 8014 +"""flagd's HTTP management port, serving ``/readyz``. + +The same endpoint the launchpad itself polls after starting flagd +(``launchpad/pkg/flagd.go``), so waiting on it here means waiting on exactly the +condition the backend considers "up". +""" + +LAUNCHPAD_PORT = 8080 +"""The testbed's control-API port. ``HttpControl`` is pointed at its mapped host port.""" + +READY_TIMEOUT_SECONDS = 60.0 +READY_POLL_SECONDS = 0.5 + + +def testbed_path() -> Path: + """Return the directory holding the testbed's compose file. + + Reaching across into the flagd package is a real coupling and is called out + where it will be read: this package has no submodule of its own, and adding + a second checkout of ``flagd-testbed`` would let the two drift to different + testbed versions -- which for a shared canonical flag set is precisely the + failure the suite is meant to detect rather than to cause. + """ + providers = Path(__file__).resolve().parents[3] + path = providers / "openfeature-provider-flagd" / "openfeature" / "test-harness" + + if not (path / "docker-compose.yaml").is_file(): + msg = ( + f"the flagd testbed is not checked out at {path}. It is a git " + f"submodule of this repository; run 'git submodule update --init' " + f"from the repository root" + ) + raise RuntimeError(msg) + return path + + +class FlagdTestbed: + """The testbed stack, and the two host ports the OFREP suite needs from it. + + Started once per session and **never restarted**: compose assigns host ports + dynamically and cannot preserve them across a restart, so a restart would + silently invalidate every provider already pointed at the old port. Scenario + isolation comes from the control API instead -- see the no-container-restart + invariant in the TCK's ``control-api.yaml``. + """ + + def __init__(self) -> None: + self._path = testbed_path() + self._version = (self._path / "version.txt").read_text().rstrip() + + # The compose file substitutes these. FLAGS_DIR is bind-mounted at + # /flags, where the launchpad writes the flag set it assembles for the + # configuration it was asked to start, so the directory has to exist + # before compose runs. A temporary one, because nothing outside the + # container reads it and a directory inside the checkout would be a + # test artifact left in the tree. + self._flags_dir = tempfile.mkdtemp(prefix="ofrep-tck-flags-") + os.environ["IMAGE"] = "ghcr.io/open-feature/flagd-testbed" + os.environ["VERSION"] = f"v{self._version}" + os.environ["FLAGS_DIR"] = self._flags_dir + + self._compose = DockerCompose( + context=str(self._path), + compose_file_name="docker-compose.yaml", + wait=True, + ) + + def start(self) -> FlagdTestbed: + self._compose.start() + self._await_ready() + return self + + def stop(self) -> None: + self._compose.stop() + + def get_ofrep_url(self) -> str: + """Return the base URL to hand to ``OFREPProvider``. + + The provider appends ``ofrep/v1/evaluate/flags/{key}`` itself + (``ofrep/__init__.py:115-119``), so this is the bare origin. + """ + return f"http://localhost:{self._mapped(OFREP_PORT)}" + + def get_launchpad_url(self) -> str: + return f"http://localhost:{self._mapped(LAUNCHPAD_PORT)}" + + def _mapped(self, port: int) -> int: + return int(self._compose.get_service_port("flagd", port)) + + def _await_ready(self) -> None: + """Block until flagd answers ``/readyz``. + + ``wait=True`` above waits for compose's own healthcheck, which polls + ``/healthz`` -- liveness, not readiness. The OFREP endpoint is only + useful once the flag sources are loaded, so this waits for the stricter + of the two rather than letting the first scenario race the load. + """ + url = f"http://localhost:{self._mapped(HEALTH_PORT)}/readyz" + deadline = time.monotonic() + READY_TIMEOUT_SECONDS + last: Exception | None = None + + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=2.0) as response: # noqa: S310 + if response.status == 200: + return + except (urllib.error.URLError, OSError) as err: # pragma: no cover + last = err + time.sleep(READY_POLL_SECONDS) + + msg = f"flagd testbed was not ready within {READY_TIMEOUT_SECONDS}s ({url})" + raise ConnectionError(msg) from last + + +def running_testbed() -> typing.Iterator[FlagdTestbed]: + """Yield a started testbed and stop it afterwards. Used by the session fixture.""" + testbed = FlagdTestbed() + testbed.start() + try: + yield testbed + finally: + testbed.stop() diff --git a/uv.lock b/uv.lock index d23aa347..fbc5606b 100644 --- a/uv.lock +++ b/uv.lock @@ -1970,9 +1970,12 @@ dependencies = [ dev = [ { name = "coverage", extra = ["toml"] }, { name = "mypy" }, + { name = "openfeature-provider-tck" }, { name = "poethepoet" }, { name = "pytest" }, + { name = "pytest-bdd" }, { name = "requests-mock" }, + { name = "testcontainers" }, { name = "types-requests" }, ] @@ -1986,9 +1989,12 @@ requires-dist = [ dev = [ { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, + { name = "openfeature-provider-tck", editable = "tools/openfeature-provider-tck" }, { name = "poethepoet", specifier = ">=0.37.0" }, { name = "pytest", specifier = ">=9.0.0,<10.0.0" }, + { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, { name = "requests-mock", specifier = ">=1.12.0,<2.0.0" }, + { name = "testcontainers", specifier = ">=4.12.0,<5.0.0" }, { name = "types-requests", specifier = ">=2.32.0,<3.0.0" }, ]