From ac48cd83c0f87e0e2845c3279fe306dbaddf95b7 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 13:30:18 +0200 Subject: [PATCH 1/7] feat(provider-tck): add the Python conformance suite for OpenFeature providers A conformance suite any Python provider can adopt to verify it implements the provider contract of the specification, and the Python implementation of the cross-language suite defined in Appendix F. It runs the same Gherkin, the same canonical flag set and the same control API as the Go and Java implementations. It uses pytest-bdd, the runner the flagd provider and the flagd testkit already use, so an adopting package gains no new test framework. Adoption is one fixture and one call. The step definitions ship as a pytest plugin registered through a pytest11 entry point, so there is no conftest.py to write and nothing to import for the vocabulary - pytest-bdd resolves steps through the fixture system, and fixtures from an installed plugin are visible everywhere. The feature files and flag set are packaged with the distribution, so adopting needs no git submodule. Capability gating uses pytest.skip from an autouse fixture, so a scenario whose capability was not declared is reported as skipped with the reason attached rather than silently passing. The gate keys off the node's markers rather than its requested fixtures: pytest-bdd resolves a step's fixtures lazily, so tck_config is not in request.fixturenames at setup time, and guarding on that silently disabled the gate. Two self-test suites, plus unit tests for what the Gherkin cannot assert about itself: the SDK's InMemoryProvider, and the TCK's own updatable one. The second exists because the first cannot exercise the configuration-change path at all. Findings, both confirmed by running the suite: * A boolean satisfies an Integer request. The client type-checks with isinstance(value, int) and bool subclasses int in Python, so boolean-flag requested as an Integer returns True with reason STATIC and no error code. This is Python-specific - the identical scenario passes in every other language - which is a fair argument for having more than one implementation. Tracked as open-feature/python-sdk#619, and marked xfail(strict=True) so it stays visible and un-hides itself once fixed. * InMemoryProvider cannot update its flag set, which Appendix A requires of an SDK in-memory provider. Only half the machinery is missing, since AbstractProvider already supplies emit_provider_configuration_changed, so ControllableInMemoryProvider is a small subclass rather than a reimplementation and should port back as a method. Tracked as open-feature/python-sdk#620. Verified locally: 56 passed, 7 skipped, 2 xfailed; ruff and mypy --strict clean. Part of https://github.com/open-feature/spec/issues/417 Signed-off-by: Simon Schrottner --- .release-please-manifest.json | 3 +- pyproject.toml | 2 + release-please-config.json | 9 + tools/openfeature-provider-tck/LICENSE | 201 ++++++++++ tools/openfeature-provider-tck/README.md | 199 ++++++++++ tools/openfeature-provider-tck/pyproject.toml | 69 ++++ .../contrib/tools/provider_tck/__init__.py | 129 ++++++ .../contrib/tools/provider_tck/capability.py | 91 +++++ .../contrib/tools/provider_tck/config.py | 169 ++++++++ .../tools/provider_tck/control-api.yaml | 368 ++++++++++++++++++ .../contrib/tools/provider_tck/control.py | 112 ++++++ .../provider_tck/features/errors.feature | 80 ++++ .../provider_tck/features/evaluation.feature | 59 +++ .../provider_tck/features/events.feature | 42 ++ .../provider_tck/features/lifecycle.feature | 33 ++ .../flag_data/canonical-flags.json | 82 ++++ .../contrib/tools/provider_tck/inprocess.py | 112 ++++++ .../contrib/tools/provider_tck/plugin.py | 104 +++++ .../contrib/tools/provider_tck/provider.py | 131 +++++++ .../contrib/tools/provider_tck/state.py | 146 +++++++ .../tools/provider_tck/steps/__init__.py | 11 + .../tools/provider_tck/steps/event_steps.py | 167 ++++++++ .../tools/provider_tck/steps/flag_steps.py | 238 +++++++++++ .../provider_tck/steps/provider_steps.py | 81 ++++ .../contrib/tools/provider_tck/values.py | 121 ++++++ .../tests/conftest.py | 37 ++ .../tests/test_controllable_conformance.py | 51 +++ .../tests/test_in_memory_conformance.py | 101 +++++ .../tests/test_in_process_control.py | 139 +++++++ 29 files changed, 3086 insertions(+), 1 deletion(-) create mode 100644 tools/openfeature-provider-tck/LICENSE create mode 100644 tools/openfeature-provider-tck/README.md create mode 100644 tools/openfeature-provider-tck/pyproject.toml create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control-api.yaml create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/errors.feature create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/evaluation.feature create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/events.feature create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/lifecycle.feature create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/flag_data/canonical-flags.json create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/__init__.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py create mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py create mode 100644 tools/openfeature-provider-tck/tests/conftest.py create mode 100644 tools/openfeature-provider-tck/tests/test_controllable_conformance.py create mode 100644 tools/openfeature-provider-tck/tests/test_in_memory_conformance.py create mode 100644 tools/openfeature-provider-tck/tests/test_in_process_control.py diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 94d58394..6bb08620 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -8,5 +8,6 @@ "providers/openfeature-provider-unleash": "0.1.2", "tools/openfeature-flagd-api": "1.0.0", "tools/openfeature-flagd-core": "1.0.0", - "tools/openfeature-flagd-api-testkit": "0.1.0" + "tools/openfeature-flagd-api-testkit": "0.1.0", + "tools/openfeature-provider-tck": "0.1.0" } diff --git a/pyproject.toml b/pyproject.toml index 647571bc..c1f1ce6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "openfeature-flagd-api", "openfeature-flagd-core", "openfeature-flagd-api-testkit", + "openfeature-provider-tck", ] [dependency-groups] @@ -43,6 +44,7 @@ openfeature-provider-unleash = { workspace = true } openfeature-flagd-api = { workspace = true } openfeature-flagd-core = { workspace = true } openfeature-flagd-api-testkit = { workspace = true } +openfeature-provider-tck = { workspace = true } [tool.uv.workspace] members = [ diff --git a/release-please-config.json b/release-please-config.json index a6335415..29cfce44 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -99,6 +99,15 @@ "extra-files": [ "README.md" ] + }, + "tools/openfeature-provider-tck": { + "package-name": "openfeature-provider-tck", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true, + "versioning": "default", + "extra-files": [ + "README.md" + ] } }, "changelog-sections": [ diff --git a/tools/openfeature-provider-tck/LICENSE b/tools/openfeature-provider-tck/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/tools/openfeature-provider-tck/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/openfeature-provider-tck/README.md b/tools/openfeature-provider-tck/README.md new file mode 100644 index 00000000..d735a5c5 --- /dev/null +++ b/tools/openfeature-provider-tck/README.md @@ -0,0 +1,199 @@ +# OpenFeature Provider TCK (Python) + +A conformance suite any OpenFeature Python provider can adopt to verify that it implements the +provider contract of the specification. + +OpenFeature's central promise is that swapping providers does not change application behaviour. +Nothing verifies that today, and every provider tests differently — so "implements the provider +contract" is an unverified claim, and a behavioural difference between two providers is discovered +by the application that trips over it. + +This package is the Python implementation of [Appendix F][appendix-f]. It runs the same Gherkin +scenarios, against the same canonical flag set, driven through the same backend control API, as +every other language's TCK. That shared basis is the point: "conformant" only means something if the +question is identical everywhere. + +Tracking issue: [open-feature/spec#417][tracking]. + +## Status + +**Proof of concept.** The scenario set is a representative subset covering each architectural +mechanism once, not exhaustive coverage. Breaking changes should be expected. + +## Adopting it + +One fixture and one call. It uses **pytest-bdd**, the same runner the flagd provider and the flagd +testkit already use, so an adopting package gains no new test framework. + +```python +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + TckConfig, + features_path, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = MyBackendControl() + return TckConfig( + name="my-provider", + control=control, + new_provider=lambda: MyProvider(control.address), + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + + +scenarios(features_path()) +``` + +There is **no `conftest.py` to write and nothing to import for the steps**. The step definitions +arrive through this package's pytest plugin, registered via a `pytest11` entry point, so installing +the package is all it takes. + +The TCK owns the whole lifecycle: registering the provider under a suite-scoped domain, awaiting +events, resetting the backend between scenarios, releasing it at the end. **If you find yourself +writing test infrastructure, that is a defect here rather than something for you to work around.** + +pytest-bdd generates one test per scenario — and one per row of a Scenario Outline — so failures +name a scenario and `-k` selects one as usual. The feature files and canonical flag set are packaged +with the distribution, so **you need no git submodule**. + +### Timings + +`TckConfig.event_timeout` is the knob that matters. Providers observe backend changes on wildly +different timescales — a streaming provider sees a configuration change in milliseconds, one that +polls every 30 seconds may need most of a poll interval. Set it to comfortably exceed your +worst-case detection latency, or the suite reports timeouts that are really just impatience. + +## Capabilities + +Not every provider implements every optional part of the contract. Each scenario exercising an +optional part carries a Gherkin tag, pytest-bdd turns that tag into a pytest marker, and a provider +declares what it supports. + +**A scenario whose capability was not declared is reported as skipped, with the reason — never as +passed.** A conformance suite that quietly goes green on scenarios it did not run is worse than no +suite at all, so `pytest.skip` carries the reason into the report: + +``` +SKIPPED provider does not declare capability @stale. + Declared: @events @object @strict-numeric-typing +``` + +| Capability | Tag | Meaning | +| --- | --- | --- | +| `Capability.EVENTS` | `@events` | emits lifecycle events at all | +| `Capability.STALE` | `@stale` | enters `STALE` and emits `PROVIDER_STALE` on backend loss | +| `Capability.CONFIGURATION_CHANGE` | `@configuration-change` | detects configuration changes and emits `PROVIDER_CONFIGURATION_CHANGED` | +| `Capability.OBJECT` | `@object` | supports structured flag values | +| `Capability.UNAVAILABLE_INIT` | `@unavailable` | reports an error state instead of hanging against a dead backend | +| `Capability.STRICT_NUMERIC_TYPING` | `@strict-numeric-typing` | does not coerce between integer and float | +| `Capability.TARGETING` | `@targeting` | reserved; no scenarios yet | +| `Capability.CACHING` | `@caching` | reserved; no scenarios yet | + +Untagged scenarios are mandatory and always run. `capabilities` defaults to everything — narrow it +rather than widening it: start from the default, run the suite, and remove only what your provider +genuinely cannot do. + +`@strict-numeric-typing` deserves a note, because unlike the others it is **not** an optional +feature. The specification requires `TYPE_MISMATCH` when the requested type cannot be satisfied, and +narrowing `0.5` to `0` loses information silently. It is a capability only so a provider with the +defect can adopt today and see the gap reported explicitly rather than being unable to adopt at all. +Not declaring it is an admission of a known bug. + +## Controlling the backend + +`BackendControl` is the single seam between the scenarios and whatever manipulates the backend. Step +definitions never talk to a backend directly, which is why the same Gherkin runs unchanged against a +containerised backend and against a provider manipulated in-process. + +**If your provider talks to a backend, drive it over the HTTP control API** — the document is +available as `control_api_spec()`. That API is the normative contract for those providers, and it is +what makes a conformance claim portable: another language's TCK drives the same endpoints against +the same stack and must get the same answers. + +Two of its requirements are easy to get wrong: + +- **Containers are never stopped or restarted mid-suite.** Unavailability is simulated *inside* the + running stack. Container orchestrators assign host ports dynamically and cannot reliably preserve + them across a restart, so restarting silently invalidates every provider already pointed at the + old port, and the failure looks like a flaky provider. +- **`/start` resets flag state; `/restart` preserves it.** An outage must be observable as a change + in availability, never as a change in flag values. + +### Providers with no backend + +An in-memory, environment-variable or file-based provider has nothing to connect to. Those may +control the backend in-process, where flag operations are direct manipulations of the provider's own +state. `InProcessControl` is the reference. + +This is a narrow allowance and the obvious thing to abuse. **A provider with an external backend +must use the control API.** Reaching into an external backend from inside the test process — a +test-only admin client, a shared database handle, a hook inside the provider — produces a suite that +passes while proving nothing, because the path it exercised is not the path the contract describes. + +Connection-dependent scenarios have no meaning without a connection, so a backend-less control +simply does not implement `ConnectionControl`, leaves `STALE` and `UNAVAILABLE_INIT` undeclared, and +those scenarios are skipped with their reason. + +## Findings + +Two, both confirmed by running the suite rather than by reading code. + +### 1. A boolean satisfies an Integer request + +`boolean-flag` evaluated through `get_integer_details` returns `True` with reason `STATIC` and **no +error code**, where the specification requires the code default and `TYPE_MISMATCH`. The client +type-checks with `isinstance(value, int)`, and `bool` is a subclass of `int` in Python. + +This is **Python-specific** — the identical scenario passes in every other language's suite, which +is a fair advertisement for having more than one implementation. Tracked as +[open-feature/python-sdk#619](https://github.com/open-feature/python-sdk/issues/619). + +The self-test marks that one row `xfail(strict=True)` with a pointer to the issue, so it stays +visible in the report and un-hides itself automatically once the SDK is fixed. + +### 2. The in-memory provider cannot update its flag set + +[Appendix A][appendix-a] requires an SDK's in-memory provider to support updating the flag set and +emitting `PROVIDER_CONFIGURATION_CHANGED`. Python's copies its mapping in the constructor and +exposes nothing to change it. Tracked as +[open-feature/python-sdk#620](https://github.com/open-feature/python-sdk/issues/620). + +Only half the machinery is missing — `AbstractProvider` already supplies +`emit_provider_configuration_changed` — which is why `ControllableInMemoryProvider` here is a small +subclass rather than a reimplementation, and why it should port back to the SDK as a method. + +## The self-tests + +| Suite | Subject | Why | +| --- | --- | --- | +| `test_in_memory_conformance` | the SDK's `InMemoryProvider` | reference adoption for a backend-less provider | +| `test_controllable_conformance` | `ControllableInMemoryProvider` | the only suite that exercises the configuration-change path — see finding 2 | +| `test_in_process_control` | `InProcessControl` | pins what the Gherkin cannot assert about itself | + +``` +56 passed, 7 skipped, 2 xfailed +``` + +No Docker, no network, under a second. + +## Known gaps + +- **The assets are vendored, not submoduled.** `features/` and `flag_data/` are copies of + `specification/assets/provider-tck/` in [open-feature/spec][spec]. Changes belong there and are + copied here; a follow-up will source them from a submodule at build time, as + `openfeature-flagd-api-testkit` already does for the flagd test harness. +- **Evaluation context passthrough is unverifiable.** The scenarios build evaluation contexts but + cannot assert one *reached* the backend. That needs an echo operation on the control API. +- **No HTTP control client yet.** It arrives with the first containerised adopter. +- **Caching, hooks and flag metadata** are not covered. + +[appendix-a]: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md +[appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md +[spec]: https://github.com/open-feature/spec +[tracking]: https://github.com/open-feature/spec/issues/417 diff --git a/tools/openfeature-provider-tck/pyproject.toml b/tools/openfeature-provider-tck/pyproject.toml new file mode 100644 index 00000000..cdddc736 --- /dev/null +++ b/tools/openfeature-provider-tck/pyproject.toml @@ -0,0 +1,69 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "openfeature-provider-tck" +version = "0.1.0" +description = "OpenFeature provider conformance suite (TCK)" +readme = "README.md" +authors = [{ name = "OpenFeature", email = "openfeature-core@groups.io" }] +license = { file = "LICENSE" } +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Framework :: Pytest", +] +keywords = ["openfeature", "conformance", "tck", "feature-flags"] +dependencies = [ + "openfeature-sdk>=0.8.2", + "pytest>=8.4.0", + # Same runner the flagd provider and the flagd testkit already use, so an + # adopting module gains no new test framework. + "pytest-bdd>=8.1.0,<9.0.0", +] +requires-python = ">=3.10" + +[project.urls] +Homepage = "https://github.com/open-feature/python-sdk-contrib" + +# Shipping the step definitions as a pytest plugin is what keeps adoption to a +# single fixture: pytest-bdd resolves steps through the fixture system, and +# fixtures from an installed plugin are visible to every test, so an adopter +# never has to `from ... import *` to pull the vocabulary in. +[project.entry-points.pytest11] +openfeature_provider_tck = "openfeature.contrib.tools.provider_tck.plugin" + +[dependency-groups] +dev = [ + "coverage[toml]>=7.10.0,<8.0.0", + "mypy>=1.18.0,<2.0.0", + "poethepoet>=0.37.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/openfeature"] + +[tool.mypy] +mypy_path = "src" +files = "src" +python_version = "3.10" +namespace_packages = true +explicit_package_bases = true +local_partial_types = true +allow_redefinition_new = true +fixed_format_cache = true +pretty = true +strict = true +disallow_any_generics = false + +[tool.coverage.run] +omit = ["tests/**"] + +[tool.poe.tasks] +test = "pytest tests" +test-cov = "coverage run -m pytest tests" +cov-report = "coverage xml" +cov = ["test-cov", "cov-report"] +mypy = "mypy" diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py new file mode 100644 index 00000000..31e9d39d --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py @@ -0,0 +1,129 @@ +"""The OpenFeature Provider Conformance Suite (TCK) for Python. + +The suite answers one question: does this provider map its backend onto the +OpenFeature provider contract correctly? It is the Python implementation of +`Appendix F`_ of the specification, and it runs the same Gherkin scenarios, +against the same canonical flag set, that every other language's TCK runs. That +shared basis is the whole point -- "conformant" only means something if the +question is identical everywhere. + +**What a provider author writes.** One fixture and one call:: + + import pytest + from pytest_bdd import scenarios + + from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, + features_path, + ) + + @pytest.fixture(scope="session") + def tck_config(): + control = InProcessControl() + return TckConfig( + name="my-provider", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + + scenarios(features_path()) + +``scenarios()`` is pytest-bdd's own, called directly rather than wrapped: it +injects the generated tests into the *calling module* by walking the stack, so a +convenience wrapper around it would deposit them inside this package instead. + +The step definitions arrive through this package's pytest plugin, so there is +nothing to import for them and no ``conftest.py`` to write. Everything else -- +registering the provider, awaiting events, resetting the backend between +scenarios, tearing down -- belongs to the TCK. If you find yourself writing test +infrastructure, that is a defect here rather than something for you to work +around. + +.. _Appendix F: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md +""" + +from __future__ import annotations + +import importlib.resources + +from .capability import ALL_CAPABILITIES, Capability +from .config import TckConfig +from .control import ( + BackendControl, + ConnectionControl, + UnsupportedControlError, +) +from .inprocess import InProcessControl +from .provider import ( + CHANGING_FLAG_KEY, + ControllableInMemoryProvider, + canonical_flag_set, +) + +__all__ = [ + "ALL_CAPABILITIES", + "CHANGING_FLAG_KEY", + "BackendControl", + "Capability", + "ConnectionControl", + "ControllableInMemoryProvider", + "InProcessControl", + "TckConfig", + "UnsupportedControlError", + "canonical_flag_set", + "canonical_flags_json", + "control_api_spec", + "features_path", +] + +# NOTE ON THE SOURCE OF TRUTH +# +# The files under features/ and flag_data/ are NOT owned by this repository. +# They are copies of the language-agnostic conformance artifacts defined in +# open-feature/spec under specification/assets/provider-tck/. They are vendored +# here so adopting this TCK never requires a git submodule of your own. Changes +# belong in open-feature/spec first and are copied here -- editing them locally +# forks the definition of conformance, which is the one thing this suite exists +# to prevent. See https://github.com/open-feature/spec/issues/417. + +_PACKAGE = "openfeature.contrib.tools.provider_tck" + + +def features_path() -> str: + """Return the directory holding the canonical feature files. + + Packaged with this distribution, so a consumer needs no submodule and no + particular directory layout. Hand it to pytest-bdd's ``scenarios()``, which + accepts an absolute path:: + + scenarios(features_path()) + + pytest-bdd generates one test per scenario -- and one per row of a Scenario + Outline -- so failures name a scenario and ``-k`` selects one as usual. + """ + return str(importlib.resources.files(_PACKAGE) / "features") + + +def canonical_flags_json() -> str: + """Return the canonical flag set as raw JSON, in the flagd flag-definition format. + + This is the flag set every scenario assumes, and a backend under test must + serve an equivalent one. The format is not what matters -- the keys, types, + variant names and resolved values are. Seed them however your backend seeds + flags. + + Exposed so an adopting provider can seed a backend from the canonical + definition rather than transcribing it, transcription being the usual way + the two drift apart. + """ + ref = importlib.resources.files(_PACKAGE) / "flag_data" / "canonical-flags.json" + return ref.read_text(encoding="utf-8") + + +def control_api_spec() -> str: + """Return the OpenAPI document a containerised backend under test must implement.""" + ref = importlib.resources.files(_PACKAGE) / "control-api.yaml" + return ref.read_text(encoding="utf-8") diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py new file mode 100644 index 00000000..04490864 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py @@ -0,0 +1,91 @@ +"""Optional parts of the provider contract, and the Gherkin tags that gate them.""" + +from __future__ import annotations + +from enum import Enum + +__all__ = ["Capability"] + + +class Capability(str, Enum): + """An optional part of the OpenFeature provider contract. + + Not every provider implements every part of the specification. A provider + backed by a static file has no meaningful notion of going stale; one with no + streaming transport cannot emit configuration-change events. Rather than + forcing such providers to fail scenarios they were never going to satisfy, + each declares what it supports through :attr:`TckConfig.capabilities`. + + Every capability corresponds to exactly one Gherkin tag. pytest-bdd turns + those tags into pytest markers, and a scenario carrying a marker whose + capability was not declared is skipped with the reason reported -- never + passed. A conformance suite that quietly goes green on scenarios it did not + run is worse than no suite at all. + + Scenarios with no capability tag are mandatory and always run. + """ + + EVENTS = "events" + """Provider emits lifecycle events at all, at minimum ``PROVIDER_READY``.""" + + STALE = "stale" + """Provider enters ``STALE`` and emits ``PROVIDER_STALE`` when it loses its backend.""" + + CONFIGURATION_CHANGE = "configuration-change" + """Provider detects configuration changes and emits ``PROVIDER_CONFIGURATION_CHANGED``.""" + + OBJECT = "object" + """Provider supports structured (object) flag values.""" + + UNAVAILABLE_INIT = "unavailable" + """Provider reports an error state promptly against a backend it cannot reach.""" + + STRICT_NUMERIC_TYPING = "strict-numeric-typing" + """Provider keeps the integer and float types distinct instead of coercing between them. + + Unlike every other entry here this is not an optional feature. The + specification requires a provider to report ``TYPE_MISMATCH`` when the + requested type cannot be satisfied, and narrowing ``0.5`` to ``0`` to satisfy + an integer request loses information silently -- the worst failure mode a + feature flag has, because the application sees a plausible value and no + error at all. + + It is a capability only so that a provider with this defect can adopt the + suite today and see the gap reported as an explicit skip, rather than being + unable to adopt at all. Not declaring it is an admission of a known bug, not + a design choice. Declare it as soon as the provider is fixed. + """ + + TARGETING = "targeting" + """Reserved. No scenario carries this tag: targeting is backend evaluation logic.""" + + CACHING = "caching" + """Reserved; no scenario carries this tag yet.""" + + @property + def tag(self) -> str: + """Return the Gherkin tag, with its leading at-sign, that gates this capability.""" + return f"@{self.value}" + + def __str__(self) -> str: + return self.tag + + +ALL_CAPABILITIES: frozenset[Capability] = frozenset(Capability) +"""Every capability the TCK recognises. + +A reasonable starting point for a new adoption: declare everything, run the +suite, and remove only what the provider genuinely cannot do. Narrowing from the +full set surfaces gaps; widening towards it hides them. +""" + +_BY_MARKER: dict[str, Capability] = {c.value: c for c in Capability} + + +def capability_for_marker(name: str) -> Capability | None: + """Map a pytest marker name onto the capability it gates, if any. + + A marker that does not name a capability gates nothing, which is what lets + the canonical feature files carry organisational tags freely. + """ + return _BY_MARKER.get(name) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py new file mode 100644 index 00000000..468a4921 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py @@ -0,0 +1,169 @@ +"""The contract a provider author implements to run the suite.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field + +from openfeature.provider import FeatureProvider + +from .capability import ALL_CAPABILITIES, Capability +from .control import BackendControl + +__all__ = ["ProviderFactory", "TckConfig"] + +ProviderFactory = Callable[[], FeatureProvider] +"""Creates the provider under test. + +A factory rather than a single instance because each scenario gets its own +provider, and because a provider often cannot be configured before the suite +starts -- a container stack's host ports do not exist until it is up. +""" + +DEFAULT_EVENT_TIMEOUT = 12.0 +DEFAULT_READY_TIMEOUT = 30.0 + + +@dataclass(frozen=True) +class TckConfig: + """Everything the TCK needs to test one provider. + + An adopting module supplies this through a session-scoped ``tck_config`` + fixture; the TCK owns everything else -- registering the provider, awaiting + events, resetting the backend between scenarios, tearing down. If you find + yourself writing test infrastructure, that is a defect in this package + rather than something for you to work around. + """ + + name: str + """Identifies the suite in test output, and scopes the OpenFeature domain + the TCK registers providers under so two suites in the same session do not + observe each other's providers. + + Use something that reads well in a failure message: ``"flagd-rpc"``, + ``"in-memory"``. + """ + + control: BackendControl + """The seam through which the TCK manipulates the backend. + + See :class:`~.control.BackendControl` for which implementation is right for + your provider. The short version: a provider with a real backend drives it + over the HTTP control API; a provider with no backend at all may control it + in-process. + """ + + new_provider: ProviderFactory + """Creates the provider under test, against a backend that is already + running and seeded with the canonical flag set. Called once per scenario. + + Return a configured but uninitialised provider; the TCK initialises it. + """ + + new_unavailable_provider: ProviderFactory | None = None + """Creates a provider pointed at a backend that does not exist. + + Used by the initialisation-failure scenarios, which assert that a provider + unable to reach its backend settles into ``ERROR`` rather than hanging or + raising out of registration. + + Point it at a closed port on localhost. Do not point it at the backend under + test -- that must stay up, and simulated outages belong to :attr:`control`. + Configure a short connection deadline: the scenario allows a bounded time + for the error, and a provider with a 30-second connect timeout will not make + it. + + Required only if :attr:`capabilities` includes + :attr:`Capability.UNAVAILABLE_INIT`. Leaving both out is the honest + configuration for a provider with no backend, and those scenarios are then + skipped with the reason reported. + """ + + capabilities: frozenset[Capability] = field(default=ALL_CAPABILITIES) + """Which optional parts of the provider contract this provider supports. + + Scenarios tagged with an undeclared capability are reported as skipped with + the reason, never as passed. Defaults to everything; narrow it rather than + widening it. + """ + + event_timeout: float = DEFAULT_EVENT_TIMEOUT + """Seconds to wait for a provider event. + + The single most important knob for a provider author, because providers + observe backend changes on wildly different timescales. A streaming provider + sees a configuration change in milliseconds; one polling every 30 seconds + may need most of a poll interval. Set it to comfortably exceed your + worst-case detection latency, or the suite reports timeouts that are really + just impatience. + + Scenarios can tighten this with the explicit ``within {int}ms`` step, which + always wins over this value. + """ + + ready_timeout: float = DEFAULT_READY_TIMEOUT + """Seconds to wait for a provider to reach ``READY`` during initialisation.""" + + def __post_init__(self) -> None: + problems: list[str] = [] + + if not self.name: + problems.append( + "name is required: it scopes the OpenFeature domain and identifies " + "the suite in test output" + ) + if self.control is None: + problems.append( + "control is required: see BackendControl for which implementation " + "fits your provider" + ) + if self.new_provider is None: + problems.append("new_provider is required: the TCK has nothing to test without it") + + # Normalise whatever iterable the caller passed into a frozenset, so a + # set literal, a list or a generator all behave the same. + object.__setattr__(self, "capabilities", frozenset(self.capabilities)) + + unknown = [c for c in self.capabilities if not isinstance(c, Capability)] + if unknown: + problems.append( + f"unknown capabilities {unknown!r}: capabilities are the members of " + f"the Capability enum" + ) + + if Capability.UNAVAILABLE_INIT in self.capabilities and self.new_unavailable_provider is None: + problems.append( + "capabilities declares Capability.UNAVAILABLE_INIT but " + "new_unavailable_provider is None: the @unavailable scenarios need a " + "provider pointed at a backend that does not exist. Supply one, or " + "remove the capability so those scenarios are skipped with a reason" + ) + + if problems: + joined = "\n - ".join(problems) + msg = f"invalid TckConfig:\n - {joined}" + raise ValueError(msg) + + @property + def domain(self) -> str: + """The OpenFeature domain this suite registers its providers under. + + Suite-scoped rather than scenario-scoped on purpose. Registering a new + provider in the same domain replaces the previous one; a fresh domain + per scenario would leave every provider of the suite registered, which + for a provider holding a network connection means leaking one connection + per scenario. + """ + return f"provider-tck/{self.name}" + + def declares(self, capability: Capability) -> bool: + return capability in self.capabilities + + @property + def sorted_capabilities(self) -> list[str]: + return sorted(c.tag for c in self.capabilities) + + +def capabilities_of(values: Iterable[Capability]) -> frozenset[Capability]: + """Convenience for building a capability set from any iterable.""" + return frozenset(values) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control-api.yaml b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control-api.yaml new file mode 100644 index 00000000..fd9bc700 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control-api.yaml @@ -0,0 +1,368 @@ +openapi: 3.0.3 + +info: + title: OpenFeature Provider TCK — Backend Control API + version: 0.0.1 + description: | + The control API that a **backend under test** must expose so the OpenFeature + Provider TCK can drive it. + + The TCK verifies the *provider contract*: how a provider maps backend + responses to typed resolution details, lifecycle states and events. To do + that it must be able to put the backend into specific states on demand — + running, unreachable, reconfigured. This document standardises how. + + This specification is derived from the control endpoints already implemented + by [`flagd-testbed`](https://github.com/open-feature/flagd-testbed)'s + "launchpad" server, which is the reference implementation. + + ## Where this document should live + + This file currently ships inside the Java `provider-tck` artifact, but it is + not a Java artifact: it is a language-agnostic contract that every language's + TCK must implement identically, and that backend vendors implement in + whatever language their testbed is written in (Go, for flagd). + + It therefore belongs in the OpenFeature **spec** repository + (`open-feature/spec`), alongside the canonical Gherkin feature files and the + canonical flag set. Those three artifacts are a single unit — a feature file + that evaluates `boolean-flag` is meaningless without the flag definition, and + a disconnect scenario is meaningless without the endpoint that produces the + disconnect. Splitting them across repositories would let them drift. + + Each language's TCK then vendors the spec repo (git submodule or equivalent) + and packages these files into its own distribution format, so that adopting a + TCK never requires a consumer to check out a submodule of their own. + + ## Conformance language + + The key words MUST, MUST NOT, REQUIRED, SHOULD, SHOULD NOT and MAY are to be + interpreted as described in RFC 2119. + + Each operation below is tagged **REQUIRED** or **OPTIONAL**. A backend that + implements every REQUIRED operation can run the full TCK. OPTIONAL operations + have a defined fallback that the TCK applies automatically, so omitting them + costs nothing but precision. + + --- + + ## Normative requirement 1 — the no-container-restart invariant + + > **Container lifecycle operations MUST NOT be used to simulate backend + > unavailability. Backend unavailability MUST be simulated from inside the + > running stack.** + + The TCK starts the vendor's Docker Compose stack **once per test suite** and + reads the dynamically mapped host ports. Testcontainers cannot reliably + preserve mapped ports across a container stop/start in all language + bindings — a restarted container generally comes back on a *different* host + port, which silently invalidates every provider instance already pointed at + the old one. Any TCK implementation in any language hits this, so the + constraint is part of the contract rather than a Java detail. + + Therefore an implementation of `/stop`, `/restart` or any other outage + simulation MUST achieve the outage by one of: + + * killing or suspending the backend **process** inside its container + (the reference behaviour — this is what flagd-testbed does); + * a proxy in the stack refusing or blackholing connections + (e.g. a toxiproxy toxic, an envoy `direct_response`); + * an in-container firewall or socket-level block. + + An implementation MUST NOT `docker stop`, `docker kill`, `docker rm` or + recreate any container in the stack while the suite is running. The stack is + brought up before the first scenario and torn down after the last one, and + the mapped ports MUST remain stable for that entire window. + + --- + + ## Normative requirement 2 — flag state semantics across outages + + Outage simulation and flag-state seeding are orthogonal, and the TCK relies + on that separation for scenario isolation: + + * `POST /start` **MUST** (re)seed flag state to the baseline defined by the + named configuration. Any mutation previously applied by `POST /change` + MUST be discarded. This is what makes `/start` usable as a reset. + * `POST /restart` and a `POST /stop` followed by a `POST /start` **of the + same configuration** MUST leave the backend serving the same baseline + flag state it served before the outage. An outage MUST NOT be observable + as a change in flag *values* — only as a change in *availability*. + * `POST /change` mutations persist until the next `/start` or `/reset`. + + --- + + ## Normative requirement 3 — compose stack conventions + + The backend under test is delivered as a **Docker Compose stack**, not a + single image, so vendors can compose proxies, edge services or several + containers. The TCK only relies on these conventions: + + * One service — by default named `backend`, overridable by the provider + author — exposes the control API on container-internal port `8080` + (also overridable). + * The same stack exposes whatever port(s) the provider connects to. + * **All external ports are dynamically mapped.** A stack MUST NOT pin host + ports; the TCK discovers them after startup and hands them to the + provider factory. + * The stack MAY contain any number of additional services. + + --- + + ## Known gap — evaluation context passthrough + + There is currently no operation for asserting that an evaluation context sent + by the provider actually reached the backend intact. Verifying that requires + an echo mechanism (e.g. `GET /last-evaluation` returning the most recent + request the backend received). Until such an operation exists, context + passthrough is out of scope for the TCK. + + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0 + +servers: + - url: http://{host}:{port} + description: | + Resolved at runtime from the Compose stack. `host` is the Docker host and + `port` is the dynamically mapped host port for the control service's + internal port 8080. + variables: + host: + default: localhost + port: + default: "8080" + +tags: + - name: lifecycle + description: Start and stop the backend process. + - name: availability + description: Simulate outages without touching containers. + - name: flags + description: Seed and mutate flag configuration. + - name: health + description: Readiness of the control API itself. + +paths: + + /start: + post: + tags: [lifecycle] + operationId: start + summary: "[REQUIRED] Start the backend and seed flags to a named baseline" + description: | + Starts the backend process using the named configuration and seeds flag + state to that configuration's baseline. + + MUST be idempotent in the sense that calling it while the backend is + already running is not an error: the implementation restarts the process + (or otherwise ensures it is running) with the requested configuration. + + Because this operation resets flag state, the TCK uses it as its default + scenario-isolation mechanism when `/reset` is not implemented. + + The set of valid configuration names is vendor-defined. Every + implementation MUST support the name `default`, which MUST serve the + canonical flag set the TCK's feature files assume. + + Reference implementation: flagd-testbed launches the `flagd` binary with + the config file of that name from `launchpad/configs` and rewrites + `/flags/allFlags.json`. + parameters: + - name: config + in: query + required: false + description: | + Name of the configuration to start with. Defaults to `default`. + schema: + type: string + default: default + example: default + responses: + "200": + description: Backend started and flag state seeded. + "400": + description: Unknown configuration name. + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + + /stop: + post: + tags: [availability] + operationId: stop + summary: "[REQUIRED] Make the backend unreachable" + description: | + Makes the backend unreachable to the provider, simulating an outage. + + **MUST NOT stop the container.** See normative requirement 1. The + reference implementation kills the flagd process while its container + keeps running. + + The backend stays unreachable until a subsequent `POST /start`. Calling + `/stop` when the backend is already stopped MUST succeed. + + The TCK uses this to drive providers into `STALE` and `ERROR` states and + to assert `PROVIDER_STALE` / `PROVIDER_ERROR` events. + responses: + "200": + description: Backend is now unreachable; container still running. + + /restart: + post: + tags: [availability] + operationId: restart + summary: "[REQUIRED] Simulate an outage of a bounded duration" + description: | + Makes the backend unreachable, waits `seconds`, then starts it again with + the configuration currently in effect. + + Flag state MUST be preserved across the outage — see normative + requirement 2. This is what distinguishes `/restart` from + `/stop` + `/start`: the former is an availability event, the latter is + also a reset. + + This operation MAY return as soon as the outage has begun rather than + blocking for the full duration; the TCK does not rely on the response + being delayed. It awaits provider events instead. + + The TCK uses this for the disconnect/reconnect scenarios: `STALE` → + `PROVIDER_STALE`, then back to `READY` → `PROVIDER_READY`. + parameters: + - name: seconds + in: query + required: false + description: | + How long the backend stays unreachable. Defaults to 5. + + Providers differ enormously in how fast they notice an outage — + a streaming provider may see it in milliseconds while a polling + provider needs up to a full poll interval. Feature files therefore + parameterise this value and provider authors tune the matching + await timeouts. + schema: + type: integer + format: int32 + minimum: 0 + default: 5 + example: 5 + responses: + "200": + description: Outage started (and, for blocking implementations, ended). + + /change: + post: + tags: [flags] + operationId: change + summary: "[REQUIRED] Mutate flag configuration so the provider observes a change" + description: | + Mutates the flag configuration such that a conforming provider observes a + configuration change and, on re-evaluation, resolves a **different value** + for the affected flag. + + The implementation MUST: + + * change the resolved value of the flag with key `changing-flag`; + * do so without restarting the backend process, so that a provider sees + a configuration-change signal rather than a reconnect; + * make the change durable until the next `/start` or `/reset`. + + The implementation SHOULD toggle between exactly two known values so that + repeated calls are meaningful and the test remains deterministic + regardless of how many times it has run against the same stack. The + reference implementation toggles `changing-flag`'s `defaultVariant` + between `foo` and `bar`. + + The TCK uses this to assert `PROVIDER_CONFIGURATION_CHANGED`, that the + changed flag key appears in the event payload, and that a subsequent + evaluation returns the new value. + responses: + "200": + description: Flag configuration mutated. + + /reset: + post: + tags: [flags] + operationId: reset + summary: "[OPTIONAL] Restore the seeded baseline without an outage" + description: | + Restores flag state to the baseline of the configuration currently in + effect, discarding any mutation applied by `/change`, **without** making + the backend unreachable at any point. + + This is the preferred scenario-isolation primitive: unlike `/start` it + causes no availability blip, so it cannot inject spurious lifecycle + events into the next scenario. + + **Scope.** This operation resets flag state only. It MUST NOT be + expected to start a backend that is currently stopped — that is what + `/start` is for. A TCK therefore uses `/reset` only when the backend is + known to be running, and `/start` otherwise. The reference client tracks + this: `/stop` and `/restart` mark the backend as possibly-unreachable, so + the scenario that follows either of them is prepared with `/start`. + + **Fallback when not implemented.** A backend that does not implement this + operation MUST respond `404` or `501`. The TCK then falls back to + `POST /start?config={defaultConfig}`, which resets flag state at the cost + of a process restart. The fallback is detected once per suite and cached. + + Implementing `/reset` is RECOMMENDED for providers whose reconnect + behaviour makes the `/start` blip hard to distinguish from a real event. + responses: + "200": + description: Flag state restored to the baseline. + "404": + description: Not implemented; the TCK falls back to `/start`. + "501": + description: Not implemented; the TCK falls back to `/start`. + + /healthz: + get: + tags: [health] + operationId: health + summary: "[OPTIONAL] Readiness of the control API" + description: | + Reports whether the control API is ready to accept commands. + + **Fallback when not implemented.** Readiness defaults to "the control + port accepts a TCP connection", which the TCK establishes with a + Testcontainers listening-port wait strategy before the first scenario. A + `404` here is therefore not a failure, and the reference implementation + does not serve this path. + + Note this reports the health of the **control API**, not of the backend. + The backend is deliberately unhealthy during outage scenarios while the + control API must stay reachable — otherwise the TCK could not end the + outage. + responses: + "200": + description: Control API ready. + content: + application/json: + schema: + $ref: "#/components/schemas/Health" + "404": + description: Not implemented; readiness falls back to a TCP port check. + "503": + description: Control API not ready yet. + +components: + schemas: + + Health: + type: object + properties: + status: + type: string + enum: [ok] + description: Present and equal to `ok` when the control API is ready. + required: [status] + + Error: + type: object + properties: + message: + type: string + description: Human-readable explanation. Never interpreted by the TCK. + required: [message] diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py new file mode 100644 index 00000000..bfa3064b --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py @@ -0,0 +1,112 @@ +"""The seam between the scenarios and whatever manipulates the backend.""" + +from __future__ import annotations + +import typing + +__all__ = [ + "BackendControl", + "ConnectionControl", + "UnsupportedControlError", + "unsupported_control", +] + + +class UnsupportedControlError(RuntimeError): + """Raised when a backend cannot perform a control operation. + + It is always a test-configuration bug rather than a provider defect. The + scenarios needing connection control are gated behind + :attr:`Capability.STALE` and :attr:`Capability.UNAVAILABLE_INIT`, so + reaching an unsupported operation means a capability was declared that the + backend cannot back up. The TCK fails loudly on it rather than skipping, + because a silent no-op would report the scenario as passed. + """ + + +@typing.runtime_checkable +class BackendControl(typing.Protocol): + """How the TCK puts the backend under test into the states a scenario needs. + + Step definitions never talk to a backend directly. They talk to this + protocol, which is why the same Gherkin runs unchanged against a + containerised backend driven over HTTP and against a provider manipulated + in-process. Nothing below this line knows about ports, containers or + transports. + + **Which implementation is right for your provider.** If your provider talks + to a backend -- a server, a service, anything out of process -- drive it + over the HTTP control API described in ``control-api.yaml``. That API is the + normative contract for those providers, and it is what makes a conformance + claim portable: another language's TCK drives the same endpoints against the + same stack and must get the same answers. + + Do not write an in-process control that reaches into an external backend + through a side channel -- a test-only admin client, a shared database + handle, a hook inside the provider. It will pass, and it will prove nothing, + because the path it exercised is not the path the contract describes. + + In-process control exists for providers with *no* backend to contract with: + in-memory, environment-variable and file-based providers, where "the + backend" is a data structure in the same process. See + :class:`InProcessControl`. + """ + + def prepare_scenario(self) -> None: + """Bring the backend to the state every scenario starts from. + + Reachable, with flag state at the baseline of the canonical flag set. + Called once before each scenario. This is the TCK's only isolation + mechanism -- scenarios share one backend for the whole suite, and + containers are never restarted between them. + """ + + def change_flag(self) -> None: + """Mutate flag configuration so a conforming provider observes a change. + + Afterwards the provider must resolve a different value for + ``changing-flag``. Which value it changes to is deliberately + unspecified; the suite asserts only that the resolved value differs. + """ + + @property + def description(self) -> str: + """A short description of what is being controlled, for messages a human reads.""" + + +@typing.runtime_checkable +class ConnectionControl(typing.Protocol): + """Implemented by a backend that can be cut off from the provider and restored. + + Separate from :class:`BackendControl` so a backend-less provider cannot + accidentally supply a no-op implementation: not implementing it at all is + the honest answer, and the TCK turns the resulting gap into an explicit, + reported skip. + """ + + def disconnect(self) -> None: + """Make the backend unreachable for the rest of the scenario, without stopping a container.""" + + def reconnect(self) -> None: + """Make the backend reachable again, preserving flag state. + + Preserving flag state is a requirement, not an implementation detail. An + outage must be observable as a change in availability and never as a + change in flag values, or the stale scenario cannot distinguish the two. + """ + + +def unsupported_control(control: BackendControl, operation: str) -> UnsupportedControlError: + """Build the error raised when a backend has no connection to control. + + The message names the fix, because the mistake it reports is always the same + one. + """ + return UnsupportedControlError( + f"{control.description} does not support {operation!r}. This is a " + f"test-configuration bug rather than a provider defect: a scenario needing " + f"connection control ran, so the suite declared Capability.STALE or " + f"Capability.UNAVAILABLE_INIT for a backend that cannot simulate an outage. " + f"Remove those capabilities from TckConfig.capabilities, or supply a " + f"BackendControl that also implements ConnectionControl." + ) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/errors.feature b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/errors.feature new file mode 100644 index 00000000..0346df3d --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/errors.feature @@ -0,0 +1,80 @@ +Feature: Provider error handling + + # Every scenario here asserts the same three-part contract, because all three parts matter and + # providers routinely get one of them wrong: + # + # 1. the code default is returned — an application must keep working, + # 2. the correct error code is reported — an application must be able to tell what went wrong, + # 3. nothing is thrown — an unhandled exception from a flag evaluation is never acceptable. + # + # Requires the backend to be seeded with the canonical flag set — see flags/canonical-flags.json. + + Background: + Given a stable provider + + Scenario Outline: Requesting the wrong type returns the code default + # The full non-numeric mismatch matrix. Numeric coercion is a separate question and is covered + # by the @strict-numeric-typing scenarios below, because "is 0.5 an integer?" has a defensible + # wrong answer whereas "is a string a boolean?" does not. + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "ERROR" + And the error-code should be "TYPE_MISMATCH" + And no exception should have been thrown + + Examples: a string flag requested as something else + | key | requested | default | + | string-flag | Boolean | false | + | string-flag | Integer | 1 | + | string-flag | Float | 0.1 | + | wrong-flag | Boolean | false | + + Examples: a boolean flag requested as something else + | key | requested | default | + | boolean-flag | String | fallback | + | boolean-flag | Integer | 1 | + | boolean-flag | Float | 0.1 | + + Examples: a numeric flag requested as a non-numeric type + | key | requested | default | + | integer-flag | Boolean | false | + | integer-flag | String | fallback | + | float-flag | Boolean | false | + | float-flag | String | fallback | + + @object + Scenario Outline: Requesting a structured flag as a scalar returns the code default + Given a -flag with key "object-flag" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "ERROR" + And the error-code should be "TYPE_MISMATCH" + And no exception should have been thrown + + Examples: + | requested | default | + | Boolean | false | + | String | fallback | + | Integer | 1 | + | Float | 0.1 | + + @strict-numeric-typing + Scenario: A float flag is not silently narrowed to an integer + # 'float-flag' resolves to 0.5. Narrowing that to an integer would lose information + # silently, so it must be reported as a type mismatch rather than rounded. + Given a Integer-flag with key "float-flag" and a default value "1" + When the flag was evaluated with details + Then the resolved details value should be "1" + And the reason should be "ERROR" + And the error-code should be "TYPE_MISMATCH" + And no exception should have been thrown + + Scenario: An unknown flag key returns the code default + # 'missing-flag' is deliberately absent from the canonical flag set. + Given a String-flag with key "missing-flag" and a default value "fallback" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + And the reason should be "ERROR" + And the error-code should be "FLAG_NOT_FOUND" + And no exception should have been thrown diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/evaluation.feature b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/evaluation.feature new file mode 100644 index 00000000..e89f174a --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/evaluation.feature @@ -0,0 +1,59 @@ +Feature: Provider flag evaluation + + # Verifies that a provider maps backend responses onto typed resolution details correctly. + # + # This does NOT test the backend's evaluation logic. Every flag in the canonical set resolves + # to its default variant with no targeting involved, so what is under test is purely the + # provider's mapping of a backend response to a value, a variant and a reason. + # + # Requires the backend to be seeded with the canonical flag set — see flags/canonical-flags.json. + + Background: + Given a stable provider + + Scenario Outline: Resolve values with variant and reason + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the variant should be "" + And the reason should be "" + And the error-code should be "" + And no exception should have been thrown + + Examples: + | key | type | default | value | variant | reason | + | boolean-flag | Boolean | false | true | on | STATIC | + | string-flag | String | bye | hi | greeting | STATIC | + | integer-flag | Integer | 1 | 10 | ten | STATIC | + | float-flag | Float | 0.1 | 0.5 | half | STATIC | + + Scenario: An integer flag resolves as an integer + # Paired with the float scenario below and with the narrowing scenario in errors.feature. + # Together they pin down that the two numeric types stay distinct rather than both being + # funnelled through one numeric representation. + Given a Integer-flag with key "integer-flag" and a default value "1" + When the flag was evaluated with details + Then the resolved details value should be "10" + And the error-code should be "" + And no exception should have been thrown + + Scenario: A float flag resolves as a float + Given a Float-flag with key "float-flag" and a default value "0.1" + When the flag was evaluated with details + Then the resolved details value should be "0.5" + And the error-code should be "" + And no exception should have been thrown + + @object + Scenario: Resolve a structured value + Given a Object-flag with key "object-flag" and a default value "{}" + When the flag was evaluated with details + Then the variant should be "template" + And the reason should be "STATIC" + And the error-code should be "" + And no exception should have been thrown + And the resolved object value should contain + | key | type | value | + | showImages | Boolean | true | + | title | String | Check out these pics! | + | imagesPerPage | Integer | 100 | diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/events.feature b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/events.feature new file mode 100644 index 00000000..00e7e5ef --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/events.feature @@ -0,0 +1,42 @@ +@events +Feature: Provider events + + # Verifies that a provider notices changes in its backend and both signals them and acts on + # them. Signalling alone is not enough: a configuration-change event that is not followed by + # a changed evaluation result is a lie, so each scenario asserts the event AND the behaviour. + # + # Outages here are simulated inside the running stack via the control API. No container is + # ever stopped or restarted — see the invariant in openapi/control-api.yaml. + + Background: + Given a stable provider + + @configuration-change + Scenario: A configuration change is signalled and applied + Given a String-flag with key "changing-flag" and a default value "unset" + And a change event handler + When the flag was evaluated with details + And the resolved value is remembered + And the flag was modified + Then the change event handler should have been executed + And the flag should be part of the event payload + When the flag was evaluated with details + Then the resolved details value should have changed + And no exception should have been thrown + + @stale + Scenario: Losing the backend makes the provider stale, regaining it makes it ready again + Given a ready event handler + And a stale event handler + When a ready event was fired + And the connection is lost + Then the stale event handler should have been executed + And the client should be in stale state + When the connection is restored + Then the ready event handler should have been executed + And the client should be in ready state + + # Deliberately NOT covered here: whether a stale provider keeps serving last-known values + # during the outage. That is caching behaviour, which depends on whether the provider holds a + # local copy of the ruleset, and it belongs behind the @caching capability once those + # scenarios are written. See the "Known gaps" section of the README. diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/lifecycle.feature b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/lifecycle.feature new file mode 100644 index 00000000..25616410 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/lifecycle.feature @@ -0,0 +1,33 @@ +@events +Feature: Provider lifecycle + + # Verifies the two terminal outcomes of provider initialisation: reaching READY against a + # healthy backend, and settling into ERROR against one that cannot be reached. + # + # The failure case matters more than it looks. A provider that blocks forever, or throws out + # of provider registration, takes the host application down with it — so the requirement is + # not merely that initialisation fails, but that it fails observably and promptly. + + Scenario: A provider reaching its backend becomes ready + Given a stable provider + And a ready event handler + Then the ready event handler should have been executed + And the client should be in ready state + + @unavailable + Scenario: A provider that cannot reach its backend reports an error + Given a unavailable provider + And a error event handler + Then the error event handler should have been executed within 10000ms + And the client should be in error state + + @unavailable + Scenario: A provider that cannot reach its backend still returns code defaults + Given a unavailable provider + And a error event handler + And a Boolean-flag with key "boolean-flag" and a default value "false" + Then the error event handler should have been executed within 10000ms + When the flag was evaluated with details + Then the resolved details value should be "false" + And the reason should be "ERROR" + And no exception should have been thrown diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/flag_data/canonical-flags.json b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/flag_data/canonical-flags.json new file mode 100644 index 00000000..343b3ae5 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/flag_data/canonical-flags.json @@ -0,0 +1,82 @@ +{ + "$comment": [ + "The canonical flag set the TCK's feature files assume. A backend under test MUST serve an", + "equivalent set under the configuration named 'default'.", + "", + "Expressed in the flagd flag-definition format because that is the only widely implemented", + "vendor-neutral format today. The format is not what matters — the keys, types, variant", + "names and resolved values are. Seed them however your backend seeds flags.", + "", + "Two things are load-bearing and easy to get wrong:", + " * 'missing-flag' MUST NOT exist. Its absence is what the FLAG_NOT_FOUND scenario tests.", + " * No flag here has targeting rules. Every scenario expects reason STATIC, because the TCK", + " tests the provider's mapping of a response, not the backend's evaluation logic." + ], + "flags": { + "boolean-flag": { + "state": "ENABLED", + "variants": { + "on": true, + "off": false + }, + "defaultVariant": "on" + }, + "string-flag": { + "state": "ENABLED", + "variants": { + "greeting": "hi", + "parting": "bye" + }, + "defaultVariant": "greeting" + }, + "integer-flag": { + "state": "ENABLED", + "variants": { + "one": 1, + "ten": 10 + }, + "defaultVariant": "ten" + }, + "float-flag": { + "state": "ENABLED", + "variants": { + "tenth": 0.1, + "half": 0.5 + }, + "defaultVariant": "half" + }, + "object-flag": { + "state": "ENABLED", + "variants": { + "empty": {}, + "template": { + "showImages": true, + "title": "Check out these pics!", + "imagesPerPage": 100 + } + }, + "defaultVariant": "template" + }, + "wrong-flag": { + "$comment": "A string flag, evaluated as a boolean by the TYPE_MISMATCH scenario.", + "state": "ENABLED", + "variants": { + "one": "uno", + "two": "dos" + }, + "defaultVariant": "one" + }, + "changing-flag": { + "$comment": [ + "The flag POST /change mutates. The TCK asserts only that its resolved value differs", + "after the change, so which of the two variants you start from does not matter." + ], + "state": "ENABLED", + "variants": { + "foo": "foo", + "bar": "bar" + }, + "defaultVariant": "foo" + } + } +} diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py new file mode 100644 index 00000000..1d69254c --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py @@ -0,0 +1,112 @@ +"""In-process backend control, for providers with no backend at all.""" + +from __future__ import annotations + +from openfeature.provider import FeatureProvider + +from .provider import ( + CHANGING_FLAG_KEY, + ControllableInMemoryProvider, + canonical_flag_set, + changing_flag, +) + +__all__ = ["InProcessControl"] + +_BASELINE = "foo" +_CHANGED = "bar" + + +class InProcessControl: + """Manipulates an in-process provider directly, with no backend and no HTTP. + + This exists so providers with nothing to connect to -- in-memory, + environment-variable and file-based providers -- can run the TCK. For those, + "the backend" is a data structure in the same process: seeding flags is + building a mapping, and changing one is an update on the live provider, so + the event the suite awaits is the provider's own + ``PROVIDER_CONFIGURATION_CHANGED`` rather than one the TCK synthesised. + + **This is not a shortcut for providers that do have a backend.** Reaching + into an external backend from inside the test process -- a test-only admin + client, a shared database handle, a hook in the provider -- produces a suite + that passes while proving nothing, because the path it exercised is not the + path the contract describes. Those providers drive the HTTP control API + instead. + + **Connection control.** :class:`InProcessControl` deliberately does not + implement :class:`~.control.ConnectionControl`. An in-memory provider has no + connection to lose, and pretending otherwise with a no-op would report the + ``@stale`` scenarios as passed. A suite using it leaves + :attr:`Capability.STALE` and :attr:`Capability.UNAVAILABLE_INIT` undeclared, + and those scenarios are skipped with the reason reported. + + **Ownership of the provider.** This type both seeds the flags and creates + the provider serving them, because in-process they are the same object: + :meth:`change_flag` has to reach the live instance to emit an event from it. + A suite therefore wires both through one control:: + + control = InProcessControl() + TckConfig( + name="in-memory", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.CONFIGURATION_CHANGE}, + ) + """ + + def __init__(self) -> None: + self._current: ControllableInMemoryProvider | None = None + self._changing_variant = _BASELINE + + @property + def description(self) -> str: + return "in-process control of an in-memory provider" + + def new_provider(self) -> FeatureProvider: + """Create the provider for the scenario about to run, at the baseline. + + Each call returns a fresh instance over a fresh copy of the canonical + flag set, which is what makes :meth:`prepare_scenario` nothing more than + dropping the previous reference. + """ + self._changing_variant = _BASELINE + self._current = ControllableInMemoryProvider(canonical_flag_set()) + return self._current + + def prepare_scenario(self) -> None: + """Drop the previous scenario's provider. + + That is the whole reset: the flag set is rebuilt per provider, so the + :meth:`new_provider` call that follows starts from an untouched + baseline. Clearing the reference rather than leaving it dangling means a + scenario that changes flags without creating a provider fails with a + clear message instead of mutating one that has already been shut down. + """ + self._current = None + + def change_flag(self) -> None: + """Flip ``changing-flag`` between its two variants on the live provider. + + The event the suite awaits is therefore the provider's own + ``PROVIDER_CONFIGURATION_CHANGED``, carrying ``changing-flag`` in + ``flags_changed``, and not a signal the TCK synthesised. + + Alternating rather than assigning a fixed variant keeps repeated calls + within one scenario meaningful; the suite asserts that the resolved + value differs, not what it became. + """ + if self._current is None: + msg = ( + "No in-memory provider exists for this scenario. In-process control " + "manipulates the provider itself, so the scenario must create one -- " + 'with "Given a stable provider" -- before any step that changes flag state.' + ) + raise RuntimeError(msg) + + self._changing_variant = ( + _BASELINE if self._changing_variant == _CHANGED else _CHANGED + ) + self._current.update_flag( + CHANGING_FLAG_KEY, changing_flag(self._changing_variant) + ) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py new file mode 100644 index 00000000..239c41ac --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py @@ -0,0 +1,104 @@ +"""The pytest plugin: capability gating, scenario state, and the shared step vocabulary. + +Registered through the ``pytest11`` entry point, so installing this package is +all it takes for the step definitions to be available. pytest-bdd resolves steps +through the fixture system and fixtures from an installed plugin are visible to +every test, which is what keeps an adoption down to one fixture and one call to +:func:`tck_scenarios`. +""" + +from __future__ import annotations + +import typing + +import pytest + +from openfeature import api + +from .capability import Capability, capability_for_marker +from .config import TckConfig +from .state import TckState + +# The step modules are registered as plugins in their own right, not merely +# imported. pytest-bdd's decorators inject a generated fixture name into the +# *defining* module's namespace, so a step is only visible to pytest once the +# module defining it is a registered plugin -- importing it here would run the +# decorators but leave those fixtures where pytest never looks. +pytest_plugins = [ + "openfeature.contrib.tools.provider_tck.steps.provider_steps", + "openfeature.contrib.tools.provider_tck.steps.flag_steps", + "openfeature.contrib.tools.provider_tck.steps.event_steps", +] + +def pytest_configure(config: pytest.Config) -> None: + """Register the capability tags as markers. + + pytest-bdd turns every Gherkin tag into a marker with + ``getattr(pytest.mark, tag)`` without registering it, which raises + ``PytestUnknownMarkWarning`` for each one -- noise at best, and a hard + failure in a project configured with ``-W error``. + """ + for capability in Capability: + config.addinivalue_line( + "markers", + f"{capability.value}: OpenFeature provider TCK capability {capability.tag}", + ) + + +@pytest.fixture +def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: + """Per-scenario state, carried between step definitions.""" + # Resetting here rather than in an autouse fixture ties the reset to the + # scenarios that actually use the TCK, and guarantees it happens after the + # capability gate has had its say -- a skipped scenario never touches the + # backend. + tck_config.control.prepare_scenario() + state = TckState(config=tck_config) + yield state + state.teardown() + + +@pytest.fixture(autouse=True) +def _tck_capability_gate(request: pytest.FixtureRequest) -> None: + """Skip a scenario whose capability the provider did not declare. + + ``pytest.skip`` here reports the scenario as skipped **with the reason**, + which is exactly what the specification asks a TCK implementation to do. + Nothing about it can be mistaken for a pass. + + The gate keys off the node's markers rather than its requested fixtures. + pytest-bdd resolves a step's fixtures lazily, as each step runs, so + ``tck_config`` is not in ``request.fixturenames`` when this autouse fixture + is set up -- guarding on that silently disabled the gate and let + ``@unavailable`` scenarios run against a config that never declared it. + + Checking markers first also means the gate costs nothing, and instantiates + nothing, for tests that are not TCK scenarios. + """ + gated = [ + capability + for marker in request.node.iter_markers() + if (capability := capability_for_marker(marker.name)) is not None + ] + if not gated: + return + + try: + config: TckConfig = request.getfixturevalue("tck_config") + except pytest.FixtureLookupError: + return + + for capability in gated: + if not config.declares(capability): + pytest.skip( + f"provider does not declare capability {capability.tag}. " + f"Declared: {' '.join(config.sorted_capabilities) or '(none)'}" + ) + + +@pytest.fixture(scope="session", autouse=True) +def _tck_release_providers() -> typing.Iterator[None]: + """Shut down whatever the suite registered once it is over.""" + yield + api.shutdown() + api.clear_providers() diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py new file mode 100644 index 00000000..5b1c9faa --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py @@ -0,0 +1,131 @@ +"""An in-memory provider that can be reconfigured at runtime, and the canonical flag set.""" + +from __future__ import annotations + +import typing + +from openfeature.event import ProviderEventDetails +from openfeature.provider.in_memory_provider import ( + FlagStorage, + InMemoryFlag, + InMemoryProvider, +) + +__all__ = [ + "CHANGING_FLAG_KEY", + "ControllableInMemoryProvider", + "canonical_flag_set", + "changing_flag", +] + +CHANGING_FLAG_KEY = "changing-flag" +"""The flag :meth:`BackendControl.change_flag` mutates.""" + +_CHANGING_BASELINE = "foo" +_CHANGING_CHANGED = "bar" + + +class ControllableInMemoryProvider(InMemoryProvider): + """An in-memory provider whose flag set can be replaced at runtime. + + **Why this exists.** `Appendix A`_ of the specification requires an SDK's + in-memory provider to "support a means of updating the ``flag set``, + resulting in the emission of ``PROVIDER_CONFIGURATION_CHANGED`` events". The + Python SDK's :class:`~openfeature.provider.in_memory_provider.InMemoryProvider` + has no such method: it copies the flag mapping in its constructor and never + exposes a way to change it. + + Only half the machinery is missing, which is what makes this a small class + rather than a reimplementation. :class:`~openfeature.provider.AbstractProvider` + already supplies ``emit_provider_configuration_changed``, and the registry + already attaches the emitter, so all that is needed is a method that swaps + the mapping and emits. Everything about *resolution* -- variants, reasons, + ``FLAG_NOT_FOUND`` -- is still the SDK's. + + That makes this an honest reference for what the SDK's provider should grow, + rather than a competing implementation that could drift from it. + + .. _Appendix A: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md + """ + + def update_flags(self, flags: FlagStorage) -> None: + """Replace the whole flag set and emit a configuration-change event. + + The event names the union of the previous and new keys, which is what + Appendix A asks for: a consumer caching evaluations needs to know + everything that might have changed, and a key that disappeared has + changed as much as one that was added. + """ + changed = sorted(set(self._flags) | set(flags)) + self._flags = dict(flags) + self.emit_provider_configuration_changed( + ProviderEventDetails( + flags_changed=changed, message="flag configuration changed" + ) + ) + + def update_flag(self, key: str, flag: InMemoryFlag[typing.Any]) -> None: + """Replace a single flag and emit a configuration-change event naming it.""" + updated = dict(self._flags) + updated[key] = flag + self._flags = updated + self.emit_provider_configuration_changed( + ProviderEventDetails( + flags_changed=[key], message="flag configuration changed" + ) + ) + + def flag(self, key: str) -> InMemoryFlag[typing.Any] | None: + """Return the flag currently registered under ``key``.""" + return self._flags.get(key) + + +def changing_flag(default_variant: str) -> InMemoryFlag[str]: + return InMemoryFlag( + default_variant=default_variant, + variants={_CHANGING_BASELINE: _CHANGING_BASELINE, _CHANGING_CHANGED: _CHANGING_CHANGED}, + ) + + +def canonical_flag_set() -> FlagStorage: + """Return the canonical flag set as SDK in-memory flags. + + Mirrors ``flag_data/canonical-flags.json`` entry for entry. Two properties + of that file are load-bearing and hold here too: + + * ``missing-flag`` is absent, which is what the ``FLAG_NOT_FOUND`` scenario + tests. Adding it turns that scenario green for the wrong reason. + * no flag carries a ``context_evaluator``, so every evaluation reports reason + ``STATIC`` -- the TCK tests a provider's mapping of a response, not a + backend's evaluation logic. + """ + return { + "boolean-flag": InMemoryFlag( + default_variant="on", variants={"on": True, "off": False} + ), + "string-flag": InMemoryFlag( + default_variant="greeting", variants={"greeting": "hi", "parting": "bye"} + ), + "integer-flag": InMemoryFlag( + default_variant="ten", variants={"one": 1, "ten": 10} + ), + "float-flag": InMemoryFlag( + default_variant="half", variants={"tenth": 0.1, "half": 0.5} + ), + "object-flag": InMemoryFlag( + default_variant="template", + variants={ + "empty": {}, + "template": { + "showImages": True, + "title": "Check out these pics!", + "imagesPerPage": 100, + }, + }, + ), + # A string flag, evaluated as a boolean by the TYPE_MISMATCH scenario. + "wrong-flag": InMemoryFlag( + default_variant="one", variants={"one": "uno", "two": "dos"} + ), + CHANGING_FLAG_KEY: changing_flag(_CHANGING_BASELINE), + } diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py new file mode 100644 index 00000000..71ea4150 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py @@ -0,0 +1,146 @@ +"""Per-scenario state: what a scenario accumulates, and how it observes events. + +Separate from :mod:`plugin` so the step modules can import these types at the top +level. The step modules are loaded by the plugin as plugins in their own right, +and a step importing from the plugin module that loads it reads like a cycle even +where it is not one. +""" + +from __future__ import annotations + +import queue +import typing +from dataclasses import dataclass, field + +from openfeature.client import OpenFeatureClient +from openfeature.event import EventDetails, ProviderEvent +from openfeature.flag_evaluation import FlagType + +from .config import TckConfig + +__all__ = ["EvaluationRecord", "EventRecorder", "TckState"] + + +@dataclass +class EvaluationRecord: + """The outcome of one flag evaluation, flattened across the five typed calls.""" + + value: typing.Any = None + variant: str | None = None + reason: str | None = None + error_code: str | None = None + error_message: str | None = None + raised: BaseException | None = None + """The exception the call raised, if any. + + In Python an errored evaluation returns the code default in the details + rather than raising, so this stays ``None`` on the error paths the suite + exercises. It is what "no exception should have been thrown" asserts. + """ + + +class EventRecorder: + """Captures the events of one type, in order, so a scenario consumes them one at a time. + + Consuming rather than merely observing is what makes the stale scenario + work: it awaits a ``PROVIDER_READY`` at the start and a second, different + ``PROVIDER_READY`` once the backend is back, and a recorder that only + remembered "ready has fired at some point" would report the second assertion + as satisfied by the first event. + + A queue rather than a list because a provider with a background thread -- + anything with a real backend -- delivers events from that thread while the + scenario waits on the main one. + """ + + def __init__(self, client: OpenFeatureClient, event: ProviderEvent) -> None: + self.event = event + self._client = client + self._events: queue.Queue[EventDetails] = queue.Queue() + self.last: EventDetails | None = None + + # The SDK replays a matching event on registration when the provider is + # already in the corresponding state, so a handler added after the + # provider became ready still observes its PROVIDER_READY. That is what + # lets the feature files register handlers after "Given a stable + # provider" without racing it. + client.add_handler(event, self._on_event) + + def _on_event(self, details: EventDetails) -> None: + self._events.put(details) + + def await_event(self, timeout: float) -> EventDetails: + """Consume the next event of this recorder's type.""" + try: + details = self._events.get(timeout=timeout) + except queue.Empty: + msg = ( + f"timed out after {timeout}s waiting for a {self.event.value} event. " + f"If the provider is simply slower than this to notice, raise " + f"TckConfig.event_timeout rather than treating it as a failure" + ) + raise AssertionError(msg) from None + self.last = details + return details + + def detach(self) -> None: + self._client.remove_handler(self.event, self._on_event) + + +@dataclass +class TckState: + """Everything one scenario accumulates.""" + + config: TckConfig + client: OpenFeatureClient | None = None + flag_key: str | None = None + flag_type: FlagType | None = None + default_value: typing.Any = None + last: EvaluationRecord | None = None + remembered: typing.Any = None + has_memory: bool = False + recorders: dict[ProviderEvent, EventRecorder] = field(default_factory=dict) + + def require_client(self) -> OpenFeatureClient: + if self.client is None: + msg = ( + "no provider has been registered in this scenario: a " + '"Given a stable provider" or "Given a unavailable provider" step ' + "must come first" + ) + raise AssertionError(msg) + return self.client + + def require_flag(self) -> tuple[str, FlagType, typing.Any]: + if self.flag_key is None or self.flag_type is None: + msg = ( + "no flag has been declared in this scenario: a " + '"Given a -flag with key ... and a default value ..." step ' + "must come first" + ) + raise AssertionError(msg) + return self.flag_key, self.flag_type, self.default_value + + def require_evaluation(self) -> EvaluationRecord: + if self.last is None: + msg = ( + "no flag has been evaluated in this scenario: a " + '"When the flag was evaluated with details" step must come first' + ) + raise AssertionError(msg) + return self.last + + def require_recorder(self, event: ProviderEvent) -> EventRecorder: + recorder = self.recorders.get(event) + if recorder is None: + msg = ( + f"no handler was registered for {event.value} in this scenario: a " + '"Given a event handler" step must come first' + ) + raise AssertionError(msg) + return recorder + + def teardown(self) -> None: + for recorder in self.recorders.values(): + recorder.detach() + self.recorders.clear() diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/__init__.py new file mode 100644 index 00000000..6581ee86 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/__init__.py @@ -0,0 +1,11 @@ +"""The shared step vocabulary. + +Each module here is registered as a pytest plugin by the TCK's own plugin, which +is what makes the steps visible: pytest-bdd's decorators inject a generated +fixture name into the *defining* module's namespace, so a step only reaches +pytest once its module is a registered plugin. + +Deliberately empty of imports. Pulling the submodules in here would import them +before pytest loads them as plugins, and pytest cannot rewrite assertions in a +module that is already imported. +""" diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py new file mode 100644 index 00000000..47a37da8 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py @@ -0,0 +1,167 @@ +"""Steps covering provider events, connection loss and client status.""" + +from __future__ import annotations + +from pytest_bdd import given, parsers, then, when + +from openfeature.event import ProviderEvent +from openfeature.provider import ProviderStatus + +from ..control import ConnectionControl, unsupported_control +from ..state import EventRecorder, TckState + +__all__ = [ + "an_event_handler", + "an_event_was_fired", + "the_client_should_be_in_state", + "the_connection_is_lost", + "the_connection_is_restored", + "the_event_handler_should_have_been_executed", + "the_event_handler_should_have_been_executed_within", + "the_flag_should_be_part_of_the_event_payload", +] + +_EVENT_BY_NAME: dict[str, ProviderEvent] = { + "ready": ProviderEvent.PROVIDER_READY, + "stale": ProviderEvent.PROVIDER_STALE, + "error": ProviderEvent.PROVIDER_ERROR, + "change": ProviderEvent.PROVIDER_CONFIGURATION_CHANGED, +} + +_STATUS_BY_NAME: dict[str, ProviderStatus] = { + "ready": ProviderStatus.READY, + "stale": ProviderStatus.STALE, + "error": ProviderStatus.ERROR, +} + + +def _event(name: str) -> ProviderEvent: + try: + return _EVENT_BY_NAME[name] + except KeyError: + msg = f"unknown event kind {name!r}" + raise AssertionError(msg) from None + + +@given(parsers.re(r"^an? (?Pready|stale|error|change) event handler$")) +def an_event_handler(tck_state: TckState, kind: str) -> None: + """Attach a recorder for one event type. + + Handlers are attached after the provider is registered, which the SDK + handles by replaying a matching event on registration when the provider is + already in the corresponding state. That is why "Given a stable provider" + followed by "And a ready event handler" is not a race. + """ + event = _event(kind) + if event in tck_state.recorders: + return + client = tck_state.require_client() + tck_state.recorders[event] = EventRecorder(client, event) + + +@when(parsers.re(r"^a (?Pready|stale|error|change) event was fired$")) +def an_event_was_fired(tck_state: TckState, kind: str) -> None: + """Consume an event, so a later assertion observes the next one rather than this. + + The stale scenario depends on it: it consumes the initial ``PROVIDER_READY`` + here and then asserts a second, distinct one once the backend is back. + """ + recorder = tck_state.require_recorder(_event(kind)) + recorder.await_event(tck_state.config.event_timeout) + + +@then( + parsers.re( + r"^the (?Pready|stale|error|change) event handler should have been executed$" + ) +) +def the_event_handler_should_have_been_executed(tck_state: TckState, kind: str) -> None: + recorder = tck_state.require_recorder(_event(kind)) + recorder.await_event(tck_state.config.event_timeout) + + +@then( + parsers.re( + r"^the (?Pready|stale|error|change) event handler should have been " + r"executed within (?P\d+)ms$" + ) +) +def the_event_handler_should_have_been_executed_within( + tck_state: TckState, kind: str, millis: str +) -> None: + """Bound the wait explicitly. + + The scenarios using this assert promptness, not merely eventual arrival: a + provider that cannot reach its backend has to report that fact quickly, + because an application blocked on provider registration is down. The bound + therefore overrides ``event_timeout`` rather than being clamped by it. + """ + recorder = tck_state.require_recorder(_event(kind)) + recorder.await_event(int(millis) / 1000.0) + + +@then("the flag should be part of the event payload") +def the_flag_should_be_part_of_the_event_payload(tck_state: TckState) -> None: + """Assert the configuration-change event named the flag that changed. + + Naming the changed flags is what makes the event actionable: a consumer + caching evaluations needs to know what to invalidate, and an event carrying + no keys forces it to invalidate everything. + """ + key, _flag_type, _default = tck_state.require_flag() + recorder = tck_state.require_recorder(ProviderEvent.PROVIDER_CONFIGURATION_CHANGED) + + if recorder.last is None: + msg = ( + "no configuration-change event has been consumed in this scenario: a " + '"the change event handler should have been executed" step must come first' + ) + raise AssertionError(msg) + + changed = recorder.last.flags_changed or [] + if key in changed: + return + + if not changed: + msg = ( + f"the configuration-change event carried no changed flags, expected it to " + f"name {key!r}" + ) + else: + msg = ( + f"the configuration-change event named {changed}, expected it to include {key!r}" + ) + raise AssertionError(msg) + + +def _connection_control(tck_state: TckState, operation: str) -> ConnectionControl: + control = tck_state.config.control + if not isinstance(control, ConnectionControl): + raise unsupported_control(control, operation) + return control + + +@when("the connection is lost") +def the_connection_is_lost(tck_state: TckState) -> None: + _connection_control(tck_state, "disconnect").disconnect() + + +@when("the connection is restored") +def the_connection_is_restored(tck_state: TckState) -> None: + _connection_control(tck_state, "reconnect").reconnect() + + +@then(parsers.re(r"^the client should be in (?Pready|stale|error) state$")) +def the_client_should_be_in_state(tck_state: TckState, name: str) -> None: + """Assert the provider status the client reports. + + Checked after the corresponding event has been consumed, and the SDK writes + provider status before running handlers, so no polling is needed: if the + event arrived, the status is already current. + """ + client = tck_state.require_client() + expected = _STATUS_BY_NAME[name] + actual = client.get_provider_status() + if actual != expected: + msg = f"client reports status {actual}, expected {expected}" + raise AssertionError(msg) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py new file mode 100644 index 00000000..f45b8dbf --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py @@ -0,0 +1,238 @@ +"""Steps that declare, evaluate and assert flags.""" + +from __future__ import annotations + +import typing +from collections.abc import Callable + +from pytest_bdd import given, parsers, then, when + +from openfeature.flag_evaluation import FlagType + +from ..state import EvaluationRecord, TckState +from ..values import describe, parse_flag_type, parse_value, values_equal + +__all__ = [ + "a_flag_with_key_and_default", + "no_exception_should_have_been_thrown", + "the_error_code_should_be", + "the_flag_was_evaluated_with_details", + "the_flag_was_modified", + "the_reason_should_be", + "the_resolved_object_value_should_contain", + "the_resolved_value_is_remembered", + "the_resolved_value_should_be", + "the_resolved_value_should_have_changed", + "the_variant_should_be", +] + + +@given( + parsers.re( + r'^an? (?P[A-Za-z]+)-flag with key "(?P[^"]*)" ' + r'and a default value "(?P[^"]*)"$' + ) +) +def a_flag_with_key_and_default( + tck_state: TckState, flag_type: str, key: str, default: str +) -> None: + """Declare the flag the scenario is about, and the type it is requested as. + + The two are independent on purpose: most of ``errors.feature`` asks for a + flag as a type it is not. + """ + parsed_type = parse_flag_type(flag_type) + tck_state.flag_key = key + tck_state.flag_type = parsed_type + tck_state.default_value = parse_value(parsed_type, default) + + +@when("the flag was evaluated with details") +def the_flag_was_evaluated_with_details(tck_state: TckState) -> None: + """Resolve the declared flag through the typed client call matching its type.""" + client = tck_state.require_client() + key, flag_type, default = tck_state.require_flag() + + # Annotated explicitly: the five typed getters have different signatures, so + # an unannotated mapping infers a value type mypy will not let us call. + calls: dict[FlagType, Callable[[str, typing.Any], typing.Any]] = { + FlagType.BOOLEAN: client.get_boolean_details, + FlagType.STRING: client.get_string_details, + FlagType.INTEGER: client.get_integer_details, + FlagType.FLOAT: client.get_float_details, + FlagType.OBJECT: client.get_object_details, + } + + record = EvaluationRecord() + try: + details = calls[flag_type](key, default) + except BaseException as exc: # recorded here, asserted on by its own step + record.raised = exc + record.value = default + else: + record.value = details.value + record.variant = details.variant + record.reason = str(details.reason) if details.reason is not None else None + record.error_code = ( + details.error_code.value if details.error_code is not None else None + ) + record.error_message = details.error_message + + tck_state.last = record + + +@then(parsers.re(r'^the resolved details value should be "(?P[^"]*)"$')) +def the_resolved_value_should_be(tck_state: TckState, expected: str) -> None: + _key, flag_type, _default = tck_state.require_flag() + record = tck_state.require_evaluation() + wanted = parse_value(flag_type, expected) + + if not values_equal(wanted, record.value): + detail = f" (the client also reported: {record.error_message})" if record.error_message else "" + msg = ( + f"flag {tck_state.flag_key!r} resolved to {describe(record.value)}, " + f"expected {describe(wanted)}{detail}" + ) + raise AssertionError(msg) + + +@then(parsers.re(r'^the variant should be "(?P[^"]*)"$')) +def the_variant_should_be(tck_state: TckState, expected: str) -> None: + record = tck_state.require_evaluation() + if record.variant != expected: + msg = ( + f"variant was {record.variant!r}, expected {expected!r}. A variant that " + f"does not survive the trip from the backend is one of the easiest parts " + f"of the contract to drop" + ) + raise AssertionError(msg) + + +@then(parsers.re(r'^the reason should be "(?P[^"]*)"$')) +def the_reason_should_be(tck_state: TckState, expected: str) -> None: + record = tck_state.require_evaluation() + if record.reason != expected: + msg = f"reason was {record.reason!r}, expected {expected!r}" + raise AssertionError(msg) + + +@then(parsers.re(r'^the error-code should be "(?P[^"]*)"$')) +def the_error_code_should_be(tck_state: TckState, expected: str) -> None: + """Assert the reported error code, where the empty string means none at all. + + The empty case matters as much as the populated ones. A provider that + reports a plausible value with no error code is the failure mode the suite + is most concerned with, because the application has no way to notice. + """ + record = tck_state.require_evaluation() + actual = record.error_code or "" + + if actual == expected: + return + + if expected == "": + msg = f"error-code was {actual!r}, expected none" + elif actual == "": + msg = ( + f"no error-code was reported, expected {expected!r}. Returning a value " + f"without an error code leaves the application unable to tell that " + f"anything went wrong" + ) + else: + msg = f"error-code was {actual!r}, expected {expected!r}" + raise AssertionError(msg) + + +@then("no exception should have been thrown") +def no_exception_should_have_been_thrown(tck_state: TckState) -> None: + """Assert the evaluation returned rather than raised. + + In Python an errored evaluation returns the code default in the details and + does not raise, so this holds on the error paths too. A provider that raises + instead takes the calling application down with it, which is what the + feature files forbid. + """ + record = tck_state.require_evaluation() + if record.raised is not None: + msg = ( + f"the evaluation raised {record.raised!r}. A flag evaluation must always " + f"return a value and an error code, never raise" + ) + raise AssertionError(msg) + + +@then("the resolved object value should contain") +def the_resolved_object_value_should_contain( + tck_state: TckState, datatable: list[list[str]] +) -> None: + """Assert members of a structured value, each with its own expected type.""" + record = tck_state.require_evaluation() + header, *rows = datatable + + if header != ["key", "type", "value"]: + msg = f"expected a data table with columns key, type, value; got {header}" + raise AssertionError(msg) + + if not isinstance(record.value, dict): + msg = ( + f"resolved object value is {describe(record.value)}, which has no members " + f"to check" + ) + raise AssertionError(msg) + + for key, raw_type, raw_value in rows: + wanted = parse_value(parse_flag_type(raw_type), raw_value) + if key not in record.value: + msg = f"resolved object value has no member {key!r}" + raise AssertionError(msg) + actual = record.value[key] + if not values_equal(wanted, actual): + msg = ( + f"object member {key!r} was {describe(actual)}, expected {describe(wanted)}" + ) + raise AssertionError(msg) + + +@when("the resolved value is remembered") +def the_resolved_value_is_remembered(tck_state: TckState) -> None: + """Store the current value so a later step can assert it changed.""" + record = tck_state.require_evaluation() + tck_state.remembered = record.value + tck_state.has_memory = True + + +@then("the resolved details value should have changed") +def the_resolved_value_should_have_changed(tck_state: TckState) -> None: + """Assert that re-evaluation produced a different value. + + This is the half of the configuration-change contract providers actually get + wrong. Emitting ``PROVIDER_CONFIGURATION_CHANGED`` and then continuing to + resolve the old value is worse than emitting nothing, because the + application acted on a signal that was not true. + """ + record = tck_state.require_evaluation() + if not tck_state.has_memory: + msg = ( + "no value was remembered in this scenario: a " + '"the resolved value is remembered" step must come first' + ) + raise AssertionError(msg) + + if values_equal(tck_state.remembered, record.value): + msg = ( + f"the resolved value is still {describe(record.value)} after the " + f"configuration changed. The change was signalled but not applied, so the " + f"event told the application something untrue" + ) + raise AssertionError(msg) + + +@when("the flag was modified") +def the_flag_was_modified(tck_state: TckState) -> None: + """Change flag configuration on the backend.""" + control = tck_state.config.control + try: + control.change_flag() + except Exception as exc: + msg = f"could not change flag configuration on {control.description}: {exc}" + raise AssertionError(msg) from exc diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py new file mode 100644 index 00000000..057b2484 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py @@ -0,0 +1,81 @@ +"""Steps that put a provider under test.""" + +from __future__ import annotations + +import contextlib + +from pytest_bdd import given, parsers + +from openfeature import api + +from ..state import TckState + +__all__ = ["a_stable_provider", "an_unavailable_provider"] + + +@given(parsers.re(r"^an? stable provider$")) +def a_stable_provider(tck_state: TckState) -> None: + """Register the provider under test against the running, seeded backend. + + ``api.set_provider`` initialises synchronously and dispatches + ``PROVIDER_READY``, so by the time this step returns the provider is ready + and every scenario that follows can assume it. A suite that started + evaluating before that would report races in the TCK as defects in the + provider. + """ + config = tck_state.config + provider = config.new_provider() + if provider is None: + msg = "TckConfig.new_provider returned None" + raise AssertionError(msg) + + try: + api.set_provider(provider, config.domain) + except Exception as exc: + msg = ( + f"registering the provider raised {exc!r}. The backend is up and seeded " + f"at this point, so this is a genuine initialisation failure rather than " + f"the unavailable-backend case" + ) + raise AssertionError(msg) from exc + + tck_state.client = api.get_client(config.domain) + + +@given(parsers.re(r"^an? unavailable provider$")) +def an_unavailable_provider(tck_state: TckState) -> None: + """Register a provider pointed at a backend that does not exist. + + Neither a failed initialisation nor a raised exception during registration + is a failure here: what the contract requires is that the provider settles + into an observable error state promptly, which the scenario asserts through + the event and the client status. The SDK's registry already converts a + raising ``initialize`` into ``PROVIDER_ERROR``, so registration itself is + expected to return normally -- but a provider that raises anyway must not + take the scenario down with it, which is why this is caught rather than + propagated. + """ + config = tck_state.config + + if config.new_unavailable_provider is None: + msg = ( + "TckConfig.new_unavailable_provider is None but an @unavailable scenario " + "ran. This is a test-configuration bug rather than a provider defect: the " + "suite declared Capability.UNAVAILABLE_INIT without supplying a provider " + "that cannot reach its backend. Remove that capability, or supply the factory" + ) + raise AssertionError(msg) + + provider = config.new_unavailable_provider() + if provider is None: + msg = "TckConfig.new_unavailable_provider returned None" + raise AssertionError(msg) + + # A raising initialize is already converted to PROVIDER_ERROR by the SDK's + # registry, so this is belt and braces: a provider that raises anyway must + # not take the scenario down with it, because the contract is about the + # observable error state rather than about how registration returned. + with contextlib.suppress(Exception): + api.set_provider(provider, config.domain) + + tck_state.client = api.get_client(config.domain) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py new file mode 100644 index 00000000..13dbe696 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py @@ -0,0 +1,121 @@ +"""Turning Gherkin strings into typed values, and comparing them with what a provider resolved.""" + +from __future__ import annotations + +import json +import typing + +from openfeature.flag_evaluation import FlagType + +__all__ = ["describe", "parse_flag_type", "parse_value", "values_equal"] + +_BY_NAME: dict[str, FlagType] = { + "boolean": FlagType.BOOLEAN, + "string": FlagType.STRING, + "integer": FlagType.INTEGER, + "float": FlagType.FLOAT, + "object": FlagType.OBJECT, +} + + +def parse_flag_type(raw: str) -> FlagType: + """Resolve the type named in a scenario, case-insensitively.""" + try: + return _BY_NAME[raw.strip().lower()] + except KeyError: + names = ", ".join(sorted(n.capitalize() for n in _BY_NAME)) + msg = f"unknown flag type {raw!r}: expected one of {names}" + raise ValueError(msg) from None + + +def _parse_bool(raw: str) -> bool: + lowered = raw.strip().lower() + if lowered in {"true", "t", "yes", "1"}: + return True + if lowered in {"false", "f", "no", "0"}: + return False + msg = f"{raw!r} is not a boolean" + raise ValueError(msg) + + +def parse_value(flag_type: FlagType, raw: str) -> typing.Any: + """Convert a value written in a scenario into the type the API uses. + + Everything in Gherkin is a string, so this is where ``"0.5"`` becomes a + float and ``"{}"`` becomes an empty object. Parsing per declared type rather + than guessing is what keeps the integer and float scenarios + distinguishable: ``"1"`` is an ``int`` in an Integer scenario and a ``float`` + in a Float one. + """ + if flag_type is FlagType.BOOLEAN: + return _parse_bool(raw) + if flag_type is FlagType.STRING: + return raw + if flag_type is FlagType.INTEGER: + return int(raw) + if flag_type is FlagType.FLOAT: + return float(raw) + if flag_type is FlagType.OBJECT: + # Gherkin escapes quotes in table cells; pytest-bdd keeps the backslash, + # so strip it before handing the text to json. + return json.loads(raw.replace('\\"', '"')) + msg = f"unknown flag type {flag_type!r}" + raise ValueError(msg) + + +def _as_number(value: typing.Any) -> float | None: + """Return a numeric value as a float, or None if it is not numeric. + + Booleans are deliberately excluded. Python makes ``bool`` a subclass of + ``int``, so an unguarded numeric comparison would quietly report ``True`` and + ``1`` as equal -- which is the exact confusion several of these scenarios + exist to detect. + """ + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + return None + + +def values_equal(expected: typing.Any, actual: typing.Any) -> bool: + """Compare an expected value from a scenario with what a provider resolved. + + Numbers are compared numerically rather than by Python type. A provider that + deserialises its backend's JSON hands back ``float`` for every number, so the + ``100`` inside ``object-flag`` arrives as ``100.0`` from one provider and + ``100`` from another while both are correct. Type distinctness is asserted + where it belongs -- by requesting a flag as a specific type and checking the + error code -- not by accident of how a number was decoded. + """ + # A boolean only ever equals a boolean. Without this, Python's bool-is-an-int + # rule would make True == 1 and quietly satisfy the scenario that exists to + # catch exactly that confusion. + if isinstance(expected, bool) or isinstance(actual, bool): + return isinstance(expected, bool) and isinstance(actual, bool) and expected == actual + + expected_number = _as_number(expected) + if expected_number is not None: + actual_number = _as_number(actual) + return actual_number is not None and expected_number == actual_number + + if isinstance(expected, dict) and isinstance(actual, dict): + if set(expected) != set(actual): + return False + return all(values_equal(v, actual[k]) for k, v in expected.items()) + + if isinstance(expected, list) and isinstance(actual, list): + return len(expected) == len(actual) and all( + values_equal(e, a) for e, a in zip(expected, actual, strict=True) + ) + + return bool(expected == actual) + + +def describe(value: typing.Any) -> str: + """Render a value for a failure message, including its type. + + "expected 100 but got 100" is the single most confusing failure a + cross-language conformance suite can produce. + """ + return f"{value!r} ({type(value).__name__})" diff --git a/tools/openfeature-provider-tck/tests/conftest.py b/tools/openfeature-provider-tck/tests/conftest.py new file mode 100644 index 00000000..3f70730f --- /dev/null +++ b/tools/openfeature-provider-tck/tests/conftest.py @@ -0,0 +1,37 @@ +"""Known deviations of the Python SDK, recorded rather than hidden. + +A conformance suite that quietly goes green on scenarios it did not run is worse +than no suite at all -- and the same is true of one that quietly goes green on a +scenario it *did* run and fail. So the one scenario the Python SDK cannot +currently satisfy is marked ``xfail(strict=True)`` here, which: + +* keeps it visible in the report, as XFAIL with the reason attached; +* fails the suite if it ever *passes*, so the marker is removed the moment the + SDK is fixed rather than lingering as a lie. + +This lives in the TCK's own self-test rather than in the shared package. It is a +fact about the SDK under test, not part of the conformance definition, and +Appendix F deliberately leaves a general "known deviations" concept as an open +question (spec#417, Q4). If that concept lands, this moves into it. +""" + +from __future__ import annotations + +import pytest + +# The Scenario Outline row that asks for boolean-flag as an Integer. +_BOOL_AS_INT = "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" + +_REASON = ( + "python-sdk: a boolean satisfies an Integer request. The client type-checks with " + "isinstance(value, int) and bool is a subclass of int in Python, so boolean-flag " + "requested as an Integer returns True with reason STATIC and no error code, where " + "the specification requires the code default and TYPE_MISMATCH. " + "See https://github.com/open-feature/python-sdk/issues/619" +) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + for item in items: + if item.name == _BOOL_AS_INT: + item.add_marker(pytest.mark.xfail(reason=_REASON, strict=True)) diff --git a/tools/openfeature-provider-tck/tests/test_controllable_conformance.py b/tools/openfeature-provider-tck/tests/test_controllable_conformance.py new file mode 100644 index 00000000..3ebd240e --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_controllable_conformance.py @@ -0,0 +1,51 @@ +"""Run the conformance suite against the TCK's own updatable in-memory provider. + +This is the suite that exercises the configuration-change path, and it exists +because the SDK's in-memory provider cannot: it has no way to update a flag set, +so ``test_in_memory_conformance`` necessarily skips those scenarios. Without +this suite the change-event step definitions would ship with no coverage at all, +and a break in them would first surface in a containerised provider suite where +it looks like a provider defect. + +It is also the reference for what an in-process control path looks like when the +provider does support updates, which is what a file-based or +environment-variable provider should be able to do. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, + features_path, +) + + +@pytest.fixture(scope="session") +def tck_config() -> TckConfig: + """Declare the provider under test and what it can do. + + ``STALE`` and ``UNAVAILABLE_INIT`` stay undeclared: there is still no + connection to lose, and ``InProcessControl`` does not implement + ``ConnectionControl``. ``CONFIGURATION_CHANGE`` is what this suite adds over + the plain in-memory one, and it is the whole point of it. + """ + control = InProcessControl() + return TckConfig( + name="controllable-in-memory", + control=control, + new_provider=control.new_provider, + capabilities={ + Capability.EVENTS, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.STRICT_NUMERIC_TYPING, + }, + ) + + +scenarios(features_path()) diff --git a/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py new file mode 100644 index 00000000..de025022 --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py @@ -0,0 +1,101 @@ +"""Run the conformance suite against the SDK's own in-memory provider. + +This is the TCK's self-test, and it earns its keep twice over. + +It is the **reference adoption** for a provider with no backend. Everything a +file-based or environment-variable provider has to write is here: one fixture +and one call. + +It is also the **Docker-free canary**. Needing no container and no network, it +runs in a fraction of a second, which makes it the fast check that catches a +broken step definition, a mis-wired capability gate or a regression in the +shared harness long before a containerised suite would. + +What it does not do is license providers that have a backend to test themselves +this way -- see ``BackendControl`` for why. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + TckConfig, + canonical_flag_set, + features_path, +) +from openfeature.provider import FeatureProvider +from openfeature.provider.in_memory_provider import InMemoryProvider + + +class PlainMemoryControl: + """Backend control for the SDK's stock in-memory provider. + + ``prepare_scenario`` is a no-op because the provider is rebuilt from the + canonical flag set for every scenario, so each one already starts from an + untouched baseline. + + ``change_flag`` cannot be implemented at all, and the error says why. + Appendix A of the specification requires an SDK's in-memory provider to + "support a means of updating the flag set, resulting in the emission of + PROVIDER_CONFIGURATION_CHANGED events"; the Python SDK's copies its mapping + in the constructor and exposes no way to change it. The suite below + therefore leaves ``CONFIGURATION_CHANGE`` undeclared and the scenario is + reported as skipped with its reason, which is the honest outcome. Reaching + this error would mean the capability had been declared anyway. + """ + + @property + def description(self) -> str: + return "the Python SDK's InMemoryProvider, rebuilt per scenario" + + def prepare_scenario(self) -> None: + return None + + def change_flag(self) -> None: + msg = ( + "openfeature.provider.in_memory_provider.InMemoryProvider cannot change its " + "flag set: it copies the mapping in its constructor and exposes no update " + "method, so a configuration change can be neither applied nor signalled. " + "Appendix A of the specification requires it. See " + "ControllableInMemoryProvider for what the SDK's provider is missing" + ) + raise NotImplementedError(msg) + + +def _new_provider() -> FeatureProvider: + return InMemoryProvider(canonical_flag_set()) + + +@pytest.fixture(scope="session") +def tck_config() -> TckConfig: + """Declare the provider under test and what it can do. + + Each omission is a fact about the provider rather than a convenience: + + * ``CONFIGURATION_CHANGE`` -- omitted because the SDK's in-memory provider + cannot update its flag set. That is a finding, not a configuration choice; + see ``PlainMemoryControl``. + * ``STALE`` and ``UNAVAILABLE_INIT`` -- omitted because there is no + connection to lose. ``PlainMemoryControl`` does not implement + ``ConnectionControl`` for the same reason, and the two omissions keep each + other honest: the scenarios are skipped before any step can reach an + operation the control cannot perform. + * ``TARGETING`` and ``CACHING`` -- omitted because no scenario carries their + tags yet, so leaving them out skips nothing. + """ + return TckConfig( + name="in-memory", + control=PlainMemoryControl(), + new_provider=_new_provider, + capabilities={ + Capability.EVENTS, + Capability.OBJECT, + Capability.STRICT_NUMERIC_TYPING, + }, + ) + + +scenarios(features_path()) diff --git a/tools/openfeature-provider-tck/tests/test_in_process_control.py b/tools/openfeature-provider-tck/tests/test_in_process_control.py new file mode 100644 index 00000000..7a100a2c --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_in_process_control.py @@ -0,0 +1,139 @@ +"""Things the Gherkin cannot assert about itself. + +Each of these is a way the in-process control path could look correct while +quietly making the conformance suites meaningless. +""" + +from __future__ import annotations + +import pytest + +from openfeature.contrib.tools.provider_tck import ( + CHANGING_FLAG_KEY, + ConnectionControl, + ControllableInMemoryProvider, + InProcessControl, + canonical_flag_set, +) +from openfeature.event import ProviderEvent + + +def _resolve_changing(provider: ControllableInMemoryProvider) -> str: + return provider.resolve_string_details(CHANGING_FLAG_KEY, "unset").value + + +def test_change_flag_actually_changes_the_resolved_value() -> None: + """The assumption every configuration-change scenario rests on. + + If ``change_flag`` emitted an event without altering what the provider + resolves, the scenario would still pass its event assertion and the suite + would be certifying a signal with nothing behind it. + """ + control = InProcessControl() + provider = control.new_provider() + assert isinstance(provider, ControllableInMemoryProvider) + + before = _resolve_changing(provider) + control.change_flag() + after = _resolve_changing(provider) + + assert before != after, "change_flag did not change the resolved value" + + +def test_change_flag_emits_a_configuration_change_event_naming_the_flag() -> None: + """The event the scenarios await is the provider's own, and it names the flag.""" + control = InProcessControl() + provider = control.new_provider() + + seen: list[tuple[ProviderEvent, list[str] | None]] = [] + + def record(_provider: object, event: ProviderEvent, details: object) -> None: + seen.append((event, getattr(details, "flags_changed", None))) + + # attach() is how the SDK registry wires a provider's emitter; doing it by + # hand keeps this a unit test of the provider rather than of the registry. + provider.attach(record) + control.change_flag() + + assert seen, "no event was emitted" + event, flags_changed = seen[-1] + assert event is ProviderEvent.PROVIDER_CONFIGURATION_CHANGED + assert flags_changed == [CHANGING_FLAG_KEY] + + +def test_change_does_not_leak_into_the_next_scenario() -> None: + """Scenario isolation. + + A leak here would make the suite order-dependent: a scenario running after + the configuration-change one would start with ``changing-flag`` already + flipped, and the failure would look like a provider defect. + """ + control = InProcessControl() + + first = control.new_provider() + assert isinstance(first, ControllableInMemoryProvider) + baseline = _resolve_changing(first) + + control.change_flag() + assert _resolve_changing(first) != baseline, "precondition: change_flag had no effect" + + control.prepare_scenario() + + second = control.new_provider() + assert isinstance(second, ControllableInMemoryProvider) + assert _resolve_changing(second) == baseline, ( + "the next scenario did not start from the baseline" + ) + + +def test_change_flag_without_a_provider_fails_clearly() -> None: + """In-process the flag store and the provider are the same object, so there is + nothing to change before one exists. Saying so beats an AttributeError.""" + control = InProcessControl() + with pytest.raises(RuntimeError, match="must create one"): + control.change_flag() + + +def test_in_process_control_does_not_pretend_to_have_a_connection() -> None: + """The load-bearing one. + + A no-op ``disconnect`` would report the ``@stale`` scenarios as passed + against a provider that cannot go stale -- precisely the silent-green + failure a conformance suite must never have. ``InProcessControl`` therefore + does not implement ``ConnectionControl`` at all, and the TCK turns that into + a skip with a reason. + """ + assert not isinstance(InProcessControl(), ConnectionControl), ( + "InProcessControl implements ConnectionControl: an in-memory provider has no " + "connection to lose, and a no-op implementation would make the @stale " + "scenarios pass without testing anything" + ) + + +def test_canonical_flag_set_omits_missing_flag() -> None: + """The property the FLAG_NOT_FOUND scenario depends on. + + Seeding ``missing-flag`` would turn that scenario green for the wrong + reason, and nothing else in the suite would notice. + """ + assert "missing-flag" not in canonical_flag_set() + + +def test_update_flags_names_the_union_of_old_and_new_keys() -> None: + """Appendix A asks for the union, not just the new keys. + + A consumer caching evaluations needs to know everything that might have + changed, and a key that disappeared has changed as much as one that arrived. + """ + provider = ControllableInMemoryProvider(canonical_flag_set()) + + seen: list[list[str] | None] = [] + provider.attach(lambda _p, _e, details: seen.append(details.flags_changed)) + + provider.update_flags({}) + + assert seen, "no event was emitted" + assert seen[-1] is not None + assert set(seen[-1]) == set(canonical_flag_set()), ( + "the event did not name every flag that disappeared" + ) From 7f281943402ef0644e280d1defe291e6c20046ea Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 13:57:43 +0200 Subject: [PATCH 2/7] fix(provider-tck): apply ruff format, and run the package in CI Two things CI caught that local verification did not. `ruff format` is a separate pre-commit hook from `ruff check`, and only the latter was run locally. Nine files needed reformatting; the changes are cosmetic line-wrapping only. More importantly, the package was not being tested in CI at all. The build matrix is gated on dorny/paths-filter and its filter list had no entry for tools/openfeature-provider-tck, so no change under that path expanded the matrix and the suite never ran. The locally reported 56 passed / 7 skipped / 2 xfailed was local-only. Adding the filter block, mirroring the one for tools/openfeature-flagd-core, turns it on. Verified after formatting: 56 passed, 7 skipped, 2 xfailed; ruff check and mypy --strict still clean. Signed-off-by: Simon Schrottner --- .github/workflows/build.yml | 3 +++ .../openfeature/contrib/tools/provider_tck/config.py | 9 +++++++-- .../openfeature/contrib/tools/provider_tck/control.py | 4 +++- .../openfeature/contrib/tools/provider_tck/plugin.py | 1 + .../openfeature/contrib/tools/provider_tck/provider.py | 5 ++++- .../contrib/tools/provider_tck/steps/event_steps.py | 4 +--- .../contrib/tools/provider_tck/steps/flag_steps.py | 10 ++++++---- .../openfeature/contrib/tools/provider_tck/values.py | 6 +++++- tools/openfeature-provider-tck/tests/conftest.py | 4 +++- .../tests/test_in_process_control.py | 4 +++- 10 files changed, 36 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8c09e514..d80adb19 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,6 +66,9 @@ jobs: tools/openfeature-flagd-api-testkit: - 'tools/openfeature-flagd-api-testkit/**' - 'uv.lock' + tools/openfeature-provider-tck: + - 'tools/openfeature-provider-tck/**' + - 'uv.lock' build: needs: changes diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py index 468a4921..b3e01fab 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py @@ -118,7 +118,9 @@ def __post_init__(self) -> None: "fits your provider" ) if self.new_provider is None: - problems.append("new_provider is required: the TCK has nothing to test without it") + problems.append( + "new_provider is required: the TCK has nothing to test without it" + ) # Normalise whatever iterable the caller passed into a frozenset, so a # set literal, a list or a generator all behave the same. @@ -131,7 +133,10 @@ def __post_init__(self) -> None: f"the Capability enum" ) - if Capability.UNAVAILABLE_INIT in self.capabilities and self.new_unavailable_provider is None: + if ( + Capability.UNAVAILABLE_INIT in self.capabilities + and self.new_unavailable_provider is None + ): problems.append( "capabilities declares Capability.UNAVAILABLE_INIT but " "new_unavailable_provider is None: the @unavailable scenarios need a " diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py index bfa3064b..0e83e5bd 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py @@ -96,7 +96,9 @@ def reconnect(self) -> None: """ -def unsupported_control(control: BackendControl, operation: str) -> UnsupportedControlError: +def unsupported_control( + control: BackendControl, operation: str +) -> UnsupportedControlError: """Build the error raised when a backend has no connection to control. The message names the fix, because the mistake it reports is always the same diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py index 239c41ac..b8b1a73b 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py @@ -30,6 +30,7 @@ "openfeature.contrib.tools.provider_tck.steps.event_steps", ] + def pytest_configure(config: pytest.Config) -> None: """Register the capability tags as markers. diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py index 5b1c9faa..33de6776 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py @@ -83,7 +83,10 @@ def flag(self, key: str) -> InMemoryFlag[typing.Any] | None: def changing_flag(default_variant: str) -> InMemoryFlag[str]: return InMemoryFlag( default_variant=default_variant, - variants={_CHANGING_BASELINE: _CHANGING_BASELINE, _CHANGING_CHANGED: _CHANGING_CHANGED}, + variants={ + _CHANGING_BASELINE: _CHANGING_BASELINE, + _CHANGING_CHANGED: _CHANGING_CHANGED, + }, ) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py index 47a37da8..6b635699 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py @@ -128,9 +128,7 @@ def the_flag_should_be_part_of_the_event_payload(tck_state: TckState) -> None: f"name {key!r}" ) else: - msg = ( - f"the configuration-change event named {changed}, expected it to include {key!r}" - ) + msg = f"the configuration-change event named {changed}, expected it to include {key!r}" raise AssertionError(msg) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py index f45b8dbf..f284eb5b 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py @@ -88,7 +88,11 @@ def the_resolved_value_should_be(tck_state: TckState, expected: str) -> None: wanted = parse_value(flag_type, expected) if not values_equal(wanted, record.value): - detail = f" (the client also reported: {record.error_message})" if record.error_message else "" + detail = ( + f" (the client also reported: {record.error_message})" + if record.error_message + else "" + ) msg = ( f"flag {tck_state.flag_key!r} resolved to {describe(record.value)}, " f"expected {describe(wanted)}{detail}" @@ -187,9 +191,7 @@ def the_resolved_object_value_should_contain( raise AssertionError(msg) actual = record.value[key] if not values_equal(wanted, actual): - msg = ( - f"object member {key!r} was {describe(actual)}, expected {describe(wanted)}" - ) + msg = f"object member {key!r} was {describe(actual)}, expected {describe(wanted)}" raise AssertionError(msg) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py index 13dbe696..4459d466 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py @@ -92,7 +92,11 @@ def values_equal(expected: typing.Any, actual: typing.Any) -> bool: # rule would make True == 1 and quietly satisfy the scenario that exists to # catch exactly that confusion. if isinstance(expected, bool) or isinstance(actual, bool): - return isinstance(expected, bool) and isinstance(actual, bool) and expected == actual + return ( + isinstance(expected, bool) + and isinstance(actual, bool) + and expected == actual + ) expected_number = _as_number(expected) if expected_number is not None: diff --git a/tools/openfeature-provider-tck/tests/conftest.py b/tools/openfeature-provider-tck/tests/conftest.py index 3f70730f..a5e6726f 100644 --- a/tools/openfeature-provider-tck/tests/conftest.py +++ b/tools/openfeature-provider-tck/tests/conftest.py @@ -20,7 +20,9 @@ import pytest # The Scenario Outline row that asks for boolean-flag as an Integer. -_BOOL_AS_INT = "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" +_BOOL_AS_INT = ( + "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" +) _REASON = ( "python-sdk: a boolean satisfies an Integer request. The client type-checks with " diff --git a/tools/openfeature-provider-tck/tests/test_in_process_control.py b/tools/openfeature-provider-tck/tests/test_in_process_control.py index 7a100a2c..88322acf 100644 --- a/tools/openfeature-provider-tck/tests/test_in_process_control.py +++ b/tools/openfeature-provider-tck/tests/test_in_process_control.py @@ -75,7 +75,9 @@ def test_change_does_not_leak_into_the_next_scenario() -> None: baseline = _resolve_changing(first) control.change_flag() - assert _resolve_changing(first) != baseline, "precondition: change_flag had no effect" + assert _resolve_changing(first) != baseline, ( + "precondition: change_flag had no effect" + ) control.prepare_scenario() From 2ffd333d95fa93258b845a9af9cff7962b6997a0 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 14:01:43 +0200 Subject: [PATCH 3/7] fix(provider-tck): add the package to uv.lock `uv sync --frozen` in the build workflow validates the lockfile against the manifests, and the previous commit added openfeature-provider-tck to the workspace root's dependencies and [tool.uv.sources] without regenerating the lock. That breaks the build job for *every* package, not just this one. It was latent until now only because the paths-filter had no entry for this package, so no build job ran at all. Enabling the filter in the previous commit would have surfaced it as a red build. The regeneration also picks up openfeature-provider-flagd 0.5.1 -> 0.5.2, which the lock had missed when that release landed. Signed-off-by: Simon Schrottner --- uv.lock | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index 3168e251..b0ab379c 100644 --- a/uv.lock +++ b/uv.lock @@ -17,6 +17,7 @@ members = [ "openfeature-provider-flagd", "openfeature-provider-flipt", "openfeature-provider-ofrep", + "openfeature-provider-tck", "openfeature-provider-unleash", "openfeature-python-contrib", ] @@ -843,7 +844,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1868,7 +1869,7 @@ dev = [ [[package]] name = "openfeature-provider-flagd" -version = "0.5.1" +version = "0.5.2" source = { editable = "providers/openfeature-provider-flagd" } dependencies = [ { name = "cachebox" }, @@ -1989,6 +1990,37 @@ dev = [ { name = "types-requests", specifier = ">=2.32.0,<3.0.0" }, ] +[[package]] +name = "openfeature-provider-tck" +version = "0.1.0" +source = { editable = "tools/openfeature-provider-tck" } +dependencies = [ + { name = "openfeature-sdk" }, + { name = "pytest" }, + { name = "pytest-bdd" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage", extra = ["toml"] }, + { name = "mypy" }, + { name = "poethepoet" }, +] + +[package.metadata] +requires-dist = [ + { name = "openfeature-sdk", specifier = ">=0.8.2" }, + { name = "pytest", specifier = ">=8.4.0" }, + { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, + { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, + { name = "poethepoet", specifier = ">=0.37.0" }, +] + [[package]] name = "openfeature-provider-unleash" version = "0.1.2" @@ -2042,6 +2074,7 @@ dependencies = [ { name = "openfeature-provider-flagd" }, { name = "openfeature-provider-flipt" }, { name = "openfeature-provider-ofrep" }, + { name = "openfeature-provider-tck" }, { name = "openfeature-provider-unleash" }, ] @@ -2063,6 +2096,7 @@ requires-dist = [ { name = "openfeature-provider-flagd", editable = "providers/openfeature-provider-flagd" }, { name = "openfeature-provider-flipt", editable = "providers/openfeature-provider-flipt" }, { name = "openfeature-provider-ofrep", editable = "providers/openfeature-provider-ofrep" }, + { name = "openfeature-provider-tck", editable = "tools/openfeature-provider-tck" }, { name = "openfeature-provider-unleash", editable = "providers/openfeature-provider-unleash" }, ] From 8324d04b3472e8e43a0dd26424186ab2d1e0aacc Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 14:02:54 +0200 Subject: [PATCH 4/7] fix(provider-tck): make ready_timeout actually bound initialisation TckConfig.ready_timeout was documented but never read by anything, so a provider that hung while connecting would hang the whole pytest session with no useful message, and the documented knob did nothing. api.set_provider initialises synchronously and has no timeout of its own, so the bound comes from running it on a worker thread and giving up on the result. The worker is deliberately not cancelled -- Python cannot interrupt a thread blocked in a socket call -- and is left to finish or die with the process, which is acceptable because a timeout already means the scenario is failing. A config field that claims to do something it does not is exactly the kind of quiet untruth this suite exists to catch, so it is fixed rather than removed. Verified: 56 passed, 7 skipped, 2 xfailed; ruff and mypy --strict clean. Signed-off-by: Simon Schrottner --- .../provider_tck/steps/provider_steps.py | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py index 057b2484..bca37fae 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py @@ -2,11 +2,13 @@ from __future__ import annotations +import concurrent.futures import contextlib from pytest_bdd import given, parsers from openfeature import api +from openfeature.provider import FeatureProvider from ..state import TckState @@ -30,7 +32,14 @@ def a_stable_provider(tck_state: TckState) -> None: raise AssertionError(msg) try: - api.set_provider(provider, config.domain) + _set_provider_within(provider, config.domain, config.ready_timeout) + except TimeoutError: + msg = ( + f"the provider did not become ready within {config.ready_timeout}s. The backend " + f"is up and seeded at this point, so either initialisation is genuinely hanging " + f"or TckConfig.ready_timeout is too short" + ) + raise AssertionError(msg) from None except Exception as exc: msg = ( f"registering the provider raised {exc!r}. The backend is up and seeded " @@ -79,3 +88,27 @@ def an_unavailable_provider(tck_state: TckState) -> None: api.set_provider(provider, config.domain) tck_state.client = api.get_client(config.domain) + + +def _set_provider_within( + provider: FeatureProvider, domain: str, timeout: float +) -> None: + """Register a provider, giving up if initialisation has not returned in time. + + ``api.set_provider`` initialises synchronously and has no timeout of its own, so a + provider that hangs while connecting would hang the whole session with no useful + message. Running it on a worker thread bounds it. + + The worker is deliberately not cancelled on timeout -- Python cannot interrupt a + thread blocked in a socket call -- so it is left to finish or die with the process. + That is acceptable here because a timeout already means the scenario is failing. + """ + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(api.set_provider, provider, domain) + try: + future.result(timeout=timeout) + except concurrent.futures.TimeoutError: + raise TimeoutError from None + finally: + # Do not block __exit__ on a worker that is still stuck. + pool.shutdown(wait=False) From 15a57bb6ec4cbaeace1db99e1dab212cea8dbf9e Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 14:12:02 +0200 Subject: [PATCH 5/7] fix(provider-tck): accept any capability collection, and type-check the tests TckConfig.capabilities was annotated frozenset[Capability], but the README tells adopters to write `capabilities={Capability.EVENTS, ...}` -- a set literal. Anyone copying the documented example and running mypy got an incompatible-argument error from the suite's own documentation. It is now annotated Collection[Capability], which is what __post_init__ already accepted: a set, a list or a generator all normalise to a frozenset on construction. The reason this was invisible is the second half of the fix. mypy was configured `files = "src"`, so the tests were never checked -- and the tests are the reference adoption, the thing an adopting provider copies. They are now in scope, which is what would have caught the annotation in the first place. Verified: mypy clean over src and tests (17 files), ruff format and check clean, 56 passed / 7 skipped / 2 xfailed. Signed-off-by: Simon Schrottner --- tools/openfeature-provider-tck/pyproject.toml | 2 +- .../src/openfeature/contrib/tools/provider_tck/config.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/openfeature-provider-tck/pyproject.toml b/tools/openfeature-provider-tck/pyproject.toml index cdddc736..3679c440 100644 --- a/tools/openfeature-provider-tck/pyproject.toml +++ b/tools/openfeature-provider-tck/pyproject.toml @@ -47,7 +47,7 @@ packages = ["src/openfeature"] [tool.mypy] mypy_path = "src" -files = "src" +files = ["src", "tests"] python_version = "3.10" namespace_packages = true explicit_package_bases = true diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py index b3e01fab..77b783cd 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Callable, Iterable +from collections.abc import Callable, Collection, Iterable from dataclasses import dataclass, field from openfeature.provider import FeatureProvider @@ -79,9 +79,14 @@ class TckConfig: skipped with the reason reported. """ - capabilities: frozenset[Capability] = field(default=ALL_CAPABILITIES) + capabilities: Collection[Capability] = field(default=ALL_CAPABILITIES) """Which optional parts of the provider contract this provider supports. + Typed as a ``Collection`` rather than a ``frozenset`` so that the obvious + thing to write -- a set literal, which is what the README shows -- is also + the correctly typed thing to write. It is normalised to a frozenset on + construction, so a list, a set or a generator all behave identically. + Scenarios tagged with an undeclared capability are reported as skipped with the reason, never as passed. Defaults to everything; narrow it rather than widening it. From 8df3b7c97fc8c0f32b6173df925fe30d287beb35 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 14:38:51 +0200 Subject: [PATCH 6/7] feat(provider-tck): source the conformance assets from the spec submodule The feature files, the canonical flag set and the control-API document are owned by open-feature/spec, not by this repository. Committing copies of them here forks the definition of conformance -- the one thing this suite exists to prevent -- and leaves no machine-checkable record of which spec revision the copies came from. Replace them with a git submodule at tools/openfeature-provider-tck/spec, pinned at dfa16586 (spec#423), plus a build-time copy. The copies are gitignored and carry a DO-NOT-EDIT marker, so the pin is now the only record of the revision and the two cannot drift apart unnoticed. An adopter installing this package still needs no submodule: the copies are force-included into the wheel and the sdist, and the sdist excludes the submodule itself so it carries the four assets rather than the whole spec repository. Only a contributor to this package needs the submodule, and `poe test` syncs it first. This mirrors what openfeature-flagd-api-testkit already does for the flagd test harness. Signed-off-by: Simon Schrottner --- .gitmodules | 3 + pyproject.toml | 5 +- tools/openfeature-provider-tck/.gitignore | 7 + tools/openfeature-provider-tck/README.md | 34 +- tools/openfeature-provider-tck/hatch_build.py | 51 +++ .../hatch_build_sync.py | 56 +++ tools/openfeature-provider-tck/pyproject.toml | 22 +- tools/openfeature-provider-tck/spec | 1 + .../contrib/tools/provider_tck/__init__.py | 23 +- .../tools/provider_tck/control-api.yaml | 368 ------------------ .../provider_tck/features/errors.feature | 80 ---- .../provider_tck/features/evaluation.feature | 59 --- .../provider_tck/features/events.feature | 42 -- .../provider_tck/features/lifecycle.feature | 33 -- .../flag_data/canonical-flags.json | 82 ---- 15 files changed, 187 insertions(+), 679 deletions(-) create mode 100644 tools/openfeature-provider-tck/.gitignore create mode 100644 tools/openfeature-provider-tck/hatch_build.py create mode 100644 tools/openfeature-provider-tck/hatch_build_sync.py create mode 160000 tools/openfeature-provider-tck/spec delete mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control-api.yaml delete mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/errors.feature delete mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/evaluation.feature delete mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/events.feature delete mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/lifecycle.feature delete mode 100644 tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/flag_data/canonical-flags.json diff --git a/.gitmodules b/.gitmodules index 7e8bf9ed..31678c42 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "providers/openfeature-provider-flagd/openfeature/test-harness"] path = providers/openfeature-provider-flagd/openfeature/test-harness url = https://github.com/open-feature/flagd-testbed.git +[submodule "tools/openfeature-provider-tck/spec"] + path = tools/openfeature-provider-tck/spec + url = https://github.com/open-feature/spec diff --git a/pyproject.toml b/pyproject.toml index c1f1ce6b..e250a4b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,10 @@ exclude = [ ".venv", "__pycache__", "venv", - "providers/openfeature-provider-flagd/src/openfeature/schemas/**" + "providers/openfeature-provider-flagd/src/openfeature/schemas/**", + # Submodules of other repositories: not ours to lint or format. + "providers/openfeature-provider-flagd/openfeature/spec/**", + "tools/openfeature-provider-tck/spec/**", ] [tool.ruff.lint] diff --git a/tools/openfeature-provider-tck/.gitignore b/tools/openfeature-provider-tck/.gitignore new file mode 100644 index 00000000..06664622 --- /dev/null +++ b/tools/openfeature-provider-tck/.gitignore @@ -0,0 +1,7 @@ +# Copied from the open-feature/spec submodule by hatch_build_sync.py. +# DO NOT EDIT the copies, and do not commit them: the canonical definitions live +# in spec/specification/assets/provider-tck/, and the revision this package is +# built against is recorded by the submodule pin. +src/openfeature/contrib/tools/provider_tck/features/ +src/openfeature/contrib/tools/provider_tck/flag_data/ +src/openfeature/contrib/tools/provider_tck/control-api.yaml diff --git a/tools/openfeature-provider-tck/README.md b/tools/openfeature-provider-tck/README.md index d735a5c5..af37eda1 100644 --- a/tools/openfeature-provider-tck/README.md +++ b/tools/openfeature-provider-tck/README.md @@ -60,7 +60,8 @@ writing test infrastructure, that is a defect here rather than something for you pytest-bdd generates one test per scenario — and one per row of a Scenario Outline — so failures name a scenario and `-k` selects one as usual. The feature files and canonical flag set are packaged -with the distribution, so **you need no git submodule**. +inside the distribution, so **adopting this package needs no git submodule** — see +[Where the assets come from](#where-the-assets-come-from). ### Timings @@ -168,6 +169,33 @@ Only half the machinery is missing — `AbstractProvider` already supplies `emit_provider_configuration_changed` — which is why `ControllableInMemoryProvider` here is a small subclass rather than a reimplementation, and why it should port back to the SDK as a method. +## Where the assets come from + +The Gherkin feature files, the canonical flag set and the control-API document are **not owned by +this repository**. They are the language-agnostic conformance artifacts defined in +[open-feature/spec][spec] under `specification/assets/provider-tck/`, and every language's TCK ships +the same ones — which is the only reason a conformance claim means the same thing in Python as it +does in Java. + +**Adopting this package needs no submodule.** The assets are copied into the wheel and the sdist at +build time, so `pip install openfeature-provider-tck` gives you everything the suite runs on. + +**Contributing to this package does.** The spec is a git submodule at +`tools/openfeature-provider-tck/spec`, and the copies under +`src/openfeature/contrib/tools/provider_tck/` are gitignored and generated: + +```bash +git submodule update --init tools/openfeature-provider-tck/spec +poe test # runs `poe sync-spec-assets` first +``` + +The copies carry a `DO-NOT-EDIT.txt` because editing them forks the definition of conformance, which +is the one thing this suite exists to prevent. A change goes to [open-feature/spec][spec] first; +then bump the submodule pin here. Committing no copies means the spec revision this package targets +is recorded by the pin and nowhere else, so the two cannot drift apart unnoticed. + +This mirrors what `openfeature-flagd-api-testkit` already does for the flagd test harness. + ## The self-tests | Suite | Subject | Why | @@ -184,10 +212,6 @@ No Docker, no network, under a second. ## Known gaps -- **The assets are vendored, not submoduled.** `features/` and `flag_data/` are copies of - `specification/assets/provider-tck/` in [open-feature/spec][spec]. Changes belong there and are - copied here; a follow-up will source them from a submodule at build time, as - `openfeature-flagd-api-testkit` already does for the flagd test harness. - **Evaluation context passthrough is unverifiable.** The scenarios build evaluation contexts but cannot assert one *reached* the backend. That needs an echo operation on the control API. - **No HTTP control client yet.** It arrives with the first containerised adopter. diff --git a/tools/openfeature-provider-tck/hatch_build.py b/tools/openfeature-provider-tck/hatch_build.py new file mode 100644 index 00000000..4b6f1d84 --- /dev/null +++ b/tools/openfeature-provider-tck/hatch_build.py @@ -0,0 +1,51 @@ +"""Hatch build hook to copy the canonical conformance assets into the package. + +The feature files, the canonical flag set and the control-API document are owned +by open-feature/spec and reach this package through a git submodule, so nothing +in this repository can fork the definition of conformance. They are copied into +the source tree at build time and force-included into the distribution, which is +what lets an *adopter* install the wheel and run the suite with no submodule of +their own. +""" + +import sys +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +# Hatchling loads this file by path rather than importing it as part of a +# package, so its directory is not on sys.path and the sibling sync module -- +# the single definition of what gets copied where -- would not be importable. +sys.path.insert(0, str(Path(__file__).parent)) + +from hatch_build_sync import FILES, PACKAGE_REL, SPEC_ASSETS, TREES, sync + + +class SpecAssetsCopyHook(BuildHookInterface): + PLUGIN_NAME = "spec-assets-copy" + + def initialize(self, version: str, build_data: dict) -> None: + root = Path(self.root) + copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + + # Building from a checkout: refresh from the submodule, so what ships is + # always the revision the pin names. Building from an sdist: there is no + # submodule, but the copies are already in the tree. + if SPEC_ASSETS.exists(): + sync() + elif not all(path.exists() for path in copies): + missing = ", ".join(str(p) for p in copies if not p.exists()) + msg = ( + f"Conformance assets missing ({missing}) and the open-feature/spec " + f"submodule is not checked out at {SPEC_ASSETS}. Run " + "`git submodule update --init tools/openfeature-provider-tck/spec`." + ) + raise FileNotFoundError(msg) + + # Force-include the gitignored copies into both sdist and wheel. + force = build_data.setdefault("force_include", {}) + for path in copies: + for member in [path] if path.is_file() else path.rglob("*"): + if member.is_file(): + rel = str(member.relative_to(root)) + force[rel] = rel diff --git a/tools/openfeature-provider-tck/hatch_build_sync.py b/tools/openfeature-provider-tck/hatch_build_sync.py new file mode 100644 index 00000000..f31bc55b --- /dev/null +++ b/tools/openfeature-provider-tck/hatch_build_sync.py @@ -0,0 +1,56 @@ +"""Copy the canonical conformance assets from the spec submodule into the package. + +Used by `poe sync-spec-assets` for local development and CI testing. The hatch +build hook (hatch_build.py) handles inclusion in the wheel and sdist. + +The assets are owned by open-feature/spec, not by this repository. Copying them +in at build time -- rather than committing copies -- means the spec revision this +package was built against is recorded by the submodule pin and nowhere else, so +the two cannot drift apart unnoticed. An *adopter* installing the wheel still +needs no submodule: the copies are inside the distribution. +""" + +import shutil +from pathlib import Path + +ROOT = Path(__file__).parent +SPEC_ASSETS = (ROOT / "spec/specification/assets/provider-tck").resolve() +PACKAGE_REL = Path("src/openfeature/contrib/tools/provider_tck") +DEST_BASE = ROOT / PACKAGE_REL + +DO_NOT_EDIT = ( + "Generated by hatch_build_sync.py from the open-feature/spec submodule.\n" + "DO NOT EDIT. Changes belong in open-feature/spec under\n" + "specification/assets/provider-tck/, then bump the submodule pin.\n" +) + +# (source directory or file, destination) relative to SPEC_ASSETS / DEST_BASE. +TREES = [("gherkin", "features"), ("flags", "flag_data")] +FILES = [("openapi/control-api.yaml", "control-api.yaml")] + + +def sync() -> None: + if not SPEC_ASSETS.exists(): + msg = ( + f"Conformance assets not found at {SPEC_ASSETS}. " + "Make sure submodules are initialized: " + "`git submodule update --init tools/openfeature-provider-tck/spec`." + ) + raise FileNotFoundError(msg) + + for src_name, dest_name in TREES: + dest = DEST_BASE / dest_name + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(SPEC_ASSETS / src_name, dest) + (dest / "DO-NOT-EDIT.txt").write_text(DO_NOT_EDIT, encoding="utf-8") + + for src_name, dest_name in FILES: + dest = DEST_BASE / dest_name + if dest.exists(): + dest.unlink() + shutil.copy2(SPEC_ASSETS / src_name, dest) + + +if __name__ == "__main__": + sync() diff --git a/tools/openfeature-provider-tck/pyproject.toml b/tools/openfeature-provider-tck/pyproject.toml index 3679c440..ff0cbe43 100644 --- a/tools/openfeature-provider-tck/pyproject.toml +++ b/tools/openfeature-provider-tck/pyproject.toml @@ -42,8 +42,25 @@ dev = [ "poethepoet>=0.37.0", ] +[tool.hatch.build.targets.sdist] +# The conformance assets are gitignored copies of the spec submodule; the build +# hook force-includes them so an sdist builds into a wheel without a submodule. +force-include = {} +# Which is why the submodule itself has no business in the sdist: it is the whole +# spec repository, and only the copies of the four assets are needed downstream. +exclude = ["/spec"] + [tool.hatch.build.targets.wheel] packages = ["src/openfeature"] +# Ship the conformance assets even though they are gitignored: an adopter +# installing this package must need no submodule of their own. +artifacts = [ + "src/openfeature/contrib/tools/provider_tck/features/", + "src/openfeature/contrib/tools/provider_tck/flag_data/", + "src/openfeature/contrib/tools/provider_tck/control-api.yaml", +] + +[tool.hatch.build.hooks.custom] [tool.mypy] mypy_path = "src" @@ -62,8 +79,9 @@ disallow_any_generics = false omit = ["tests/**"] [tool.poe.tasks] -test = "pytest tests" -test-cov = "coverage run -m pytest tests" +sync-spec-assets = "python hatch_build_sync.py" +test = ["sync-spec-assets", {cmd = "pytest tests"}] +test-cov = ["sync-spec-assets", {cmd = "coverage run -m pytest tests"}] cov-report = "coverage xml" cov = ["test-cov", "cov-report"] mypy = "mypy" diff --git a/tools/openfeature-provider-tck/spec b/tools/openfeature-provider-tck/spec new file mode 160000 index 00000000..dfa16586 --- /dev/null +++ b/tools/openfeature-provider-tck/spec @@ -0,0 +1 @@ +Subproject commit dfa16586d91ca020ef1b3b82a7c972d833ff8f29 diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py index 31e9d39d..8b615296 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py @@ -81,13 +81,22 @@ def tck_config(): # NOTE ON THE SOURCE OF TRUTH # -# The files under features/ and flag_data/ are NOT owned by this repository. -# They are copies of the language-agnostic conformance artifacts defined in -# open-feature/spec under specification/assets/provider-tck/. They are vendored -# here so adopting this TCK never requires a git submodule of your own. Changes -# belong in open-feature/spec first and are copied here -- editing them locally -# forks the definition of conformance, which is the one thing this suite exists -# to prevent. See https://github.com/open-feature/spec/issues/417. +# The files under features/ and flag_data/, and control-api.yaml, are NOT owned +# by this repository and are NOT committed to it. They are copies of the +# language-agnostic conformance artifacts defined in open-feature/spec under +# specification/assets/provider-tck/, which reaches this package as a git +# submodule at tools/openfeature-provider-tck/spec and is copied in at build +# time by hatch_build.py. The copies are gitignored, so the only record of which +# spec revision this package targets is the submodule pin, and the two cannot +# drift apart unnoticed. +# +# They are copied into the distribution, so an adopter installing this package +# needs no submodule of their own; only a contributor to this package does. +# +# Changes belong in open-feature/spec first, followed by a bump of the submodule +# pin -- editing the copies locally forks the definition of conformance, which is +# the one thing this suite exists to prevent. +# See https://github.com/open-feature/spec/issues/417. _PACKAGE = "openfeature.contrib.tools.provider_tck" diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control-api.yaml b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control-api.yaml deleted file mode 100644 index fd9bc700..00000000 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control-api.yaml +++ /dev/null @@ -1,368 +0,0 @@ -openapi: 3.0.3 - -info: - title: OpenFeature Provider TCK — Backend Control API - version: 0.0.1 - description: | - The control API that a **backend under test** must expose so the OpenFeature - Provider TCK can drive it. - - The TCK verifies the *provider contract*: how a provider maps backend - responses to typed resolution details, lifecycle states and events. To do - that it must be able to put the backend into specific states on demand — - running, unreachable, reconfigured. This document standardises how. - - This specification is derived from the control endpoints already implemented - by [`flagd-testbed`](https://github.com/open-feature/flagd-testbed)'s - "launchpad" server, which is the reference implementation. - - ## Where this document should live - - This file currently ships inside the Java `provider-tck` artifact, but it is - not a Java artifact: it is a language-agnostic contract that every language's - TCK must implement identically, and that backend vendors implement in - whatever language their testbed is written in (Go, for flagd). - - It therefore belongs in the OpenFeature **spec** repository - (`open-feature/spec`), alongside the canonical Gherkin feature files and the - canonical flag set. Those three artifacts are a single unit — a feature file - that evaluates `boolean-flag` is meaningless without the flag definition, and - a disconnect scenario is meaningless without the endpoint that produces the - disconnect. Splitting them across repositories would let them drift. - - Each language's TCK then vendors the spec repo (git submodule or equivalent) - and packages these files into its own distribution format, so that adopting a - TCK never requires a consumer to check out a submodule of their own. - - ## Conformance language - - The key words MUST, MUST NOT, REQUIRED, SHOULD, SHOULD NOT and MAY are to be - interpreted as described in RFC 2119. - - Each operation below is tagged **REQUIRED** or **OPTIONAL**. A backend that - implements every REQUIRED operation can run the full TCK. OPTIONAL operations - have a defined fallback that the TCK applies automatically, so omitting them - costs nothing but precision. - - --- - - ## Normative requirement 1 — the no-container-restart invariant - - > **Container lifecycle operations MUST NOT be used to simulate backend - > unavailability. Backend unavailability MUST be simulated from inside the - > running stack.** - - The TCK starts the vendor's Docker Compose stack **once per test suite** and - reads the dynamically mapped host ports. Testcontainers cannot reliably - preserve mapped ports across a container stop/start in all language - bindings — a restarted container generally comes back on a *different* host - port, which silently invalidates every provider instance already pointed at - the old one. Any TCK implementation in any language hits this, so the - constraint is part of the contract rather than a Java detail. - - Therefore an implementation of `/stop`, `/restart` or any other outage - simulation MUST achieve the outage by one of: - - * killing or suspending the backend **process** inside its container - (the reference behaviour — this is what flagd-testbed does); - * a proxy in the stack refusing or blackholing connections - (e.g. a toxiproxy toxic, an envoy `direct_response`); - * an in-container firewall or socket-level block. - - An implementation MUST NOT `docker stop`, `docker kill`, `docker rm` or - recreate any container in the stack while the suite is running. The stack is - brought up before the first scenario and torn down after the last one, and - the mapped ports MUST remain stable for that entire window. - - --- - - ## Normative requirement 2 — flag state semantics across outages - - Outage simulation and flag-state seeding are orthogonal, and the TCK relies - on that separation for scenario isolation: - - * `POST /start` **MUST** (re)seed flag state to the baseline defined by the - named configuration. Any mutation previously applied by `POST /change` - MUST be discarded. This is what makes `/start` usable as a reset. - * `POST /restart` and a `POST /stop` followed by a `POST /start` **of the - same configuration** MUST leave the backend serving the same baseline - flag state it served before the outage. An outage MUST NOT be observable - as a change in flag *values* — only as a change in *availability*. - * `POST /change` mutations persist until the next `/start` or `/reset`. - - --- - - ## Normative requirement 3 — compose stack conventions - - The backend under test is delivered as a **Docker Compose stack**, not a - single image, so vendors can compose proxies, edge services or several - containers. The TCK only relies on these conventions: - - * One service — by default named `backend`, overridable by the provider - author — exposes the control API on container-internal port `8080` - (also overridable). - * The same stack exposes whatever port(s) the provider connects to. - * **All external ports are dynamically mapped.** A stack MUST NOT pin host - ports; the TCK discovers them after startup and hands them to the - provider factory. - * The stack MAY contain any number of additional services. - - --- - - ## Known gap — evaluation context passthrough - - There is currently no operation for asserting that an evaluation context sent - by the provider actually reached the backend intact. Verifying that requires - an echo mechanism (e.g. `GET /last-evaluation` returning the most recent - request the backend received). Until such an operation exists, context - passthrough is out of scope for the TCK. - - license: - name: Apache 2.0 - url: https://www.apache.org/licenses/LICENSE-2.0 - -servers: - - url: http://{host}:{port} - description: | - Resolved at runtime from the Compose stack. `host` is the Docker host and - `port` is the dynamically mapped host port for the control service's - internal port 8080. - variables: - host: - default: localhost - port: - default: "8080" - -tags: - - name: lifecycle - description: Start and stop the backend process. - - name: availability - description: Simulate outages without touching containers. - - name: flags - description: Seed and mutate flag configuration. - - name: health - description: Readiness of the control API itself. - -paths: - - /start: - post: - tags: [lifecycle] - operationId: start - summary: "[REQUIRED] Start the backend and seed flags to a named baseline" - description: | - Starts the backend process using the named configuration and seeds flag - state to that configuration's baseline. - - MUST be idempotent in the sense that calling it while the backend is - already running is not an error: the implementation restarts the process - (or otherwise ensures it is running) with the requested configuration. - - Because this operation resets flag state, the TCK uses it as its default - scenario-isolation mechanism when `/reset` is not implemented. - - The set of valid configuration names is vendor-defined. Every - implementation MUST support the name `default`, which MUST serve the - canonical flag set the TCK's feature files assume. - - Reference implementation: flagd-testbed launches the `flagd` binary with - the config file of that name from `launchpad/configs` and rewrites - `/flags/allFlags.json`. - parameters: - - name: config - in: query - required: false - description: | - Name of the configuration to start with. Defaults to `default`. - schema: - type: string - default: default - example: default - responses: - "200": - description: Backend started and flag state seeded. - "400": - description: Unknown configuration name. - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - - /stop: - post: - tags: [availability] - operationId: stop - summary: "[REQUIRED] Make the backend unreachable" - description: | - Makes the backend unreachable to the provider, simulating an outage. - - **MUST NOT stop the container.** See normative requirement 1. The - reference implementation kills the flagd process while its container - keeps running. - - The backend stays unreachable until a subsequent `POST /start`. Calling - `/stop` when the backend is already stopped MUST succeed. - - The TCK uses this to drive providers into `STALE` and `ERROR` states and - to assert `PROVIDER_STALE` / `PROVIDER_ERROR` events. - responses: - "200": - description: Backend is now unreachable; container still running. - - /restart: - post: - tags: [availability] - operationId: restart - summary: "[REQUIRED] Simulate an outage of a bounded duration" - description: | - Makes the backend unreachable, waits `seconds`, then starts it again with - the configuration currently in effect. - - Flag state MUST be preserved across the outage — see normative - requirement 2. This is what distinguishes `/restart` from - `/stop` + `/start`: the former is an availability event, the latter is - also a reset. - - This operation MAY return as soon as the outage has begun rather than - blocking for the full duration; the TCK does not rely on the response - being delayed. It awaits provider events instead. - - The TCK uses this for the disconnect/reconnect scenarios: `STALE` → - `PROVIDER_STALE`, then back to `READY` → `PROVIDER_READY`. - parameters: - - name: seconds - in: query - required: false - description: | - How long the backend stays unreachable. Defaults to 5. - - Providers differ enormously in how fast they notice an outage — - a streaming provider may see it in milliseconds while a polling - provider needs up to a full poll interval. Feature files therefore - parameterise this value and provider authors tune the matching - await timeouts. - schema: - type: integer - format: int32 - minimum: 0 - default: 5 - example: 5 - responses: - "200": - description: Outage started (and, for blocking implementations, ended). - - /change: - post: - tags: [flags] - operationId: change - summary: "[REQUIRED] Mutate flag configuration so the provider observes a change" - description: | - Mutates the flag configuration such that a conforming provider observes a - configuration change and, on re-evaluation, resolves a **different value** - for the affected flag. - - The implementation MUST: - - * change the resolved value of the flag with key `changing-flag`; - * do so without restarting the backend process, so that a provider sees - a configuration-change signal rather than a reconnect; - * make the change durable until the next `/start` or `/reset`. - - The implementation SHOULD toggle between exactly two known values so that - repeated calls are meaningful and the test remains deterministic - regardless of how many times it has run against the same stack. The - reference implementation toggles `changing-flag`'s `defaultVariant` - between `foo` and `bar`. - - The TCK uses this to assert `PROVIDER_CONFIGURATION_CHANGED`, that the - changed flag key appears in the event payload, and that a subsequent - evaluation returns the new value. - responses: - "200": - description: Flag configuration mutated. - - /reset: - post: - tags: [flags] - operationId: reset - summary: "[OPTIONAL] Restore the seeded baseline without an outage" - description: | - Restores flag state to the baseline of the configuration currently in - effect, discarding any mutation applied by `/change`, **without** making - the backend unreachable at any point. - - This is the preferred scenario-isolation primitive: unlike `/start` it - causes no availability blip, so it cannot inject spurious lifecycle - events into the next scenario. - - **Scope.** This operation resets flag state only. It MUST NOT be - expected to start a backend that is currently stopped — that is what - `/start` is for. A TCK therefore uses `/reset` only when the backend is - known to be running, and `/start` otherwise. The reference client tracks - this: `/stop` and `/restart` mark the backend as possibly-unreachable, so - the scenario that follows either of them is prepared with `/start`. - - **Fallback when not implemented.** A backend that does not implement this - operation MUST respond `404` or `501`. The TCK then falls back to - `POST /start?config={defaultConfig}`, which resets flag state at the cost - of a process restart. The fallback is detected once per suite and cached. - - Implementing `/reset` is RECOMMENDED for providers whose reconnect - behaviour makes the `/start` blip hard to distinguish from a real event. - responses: - "200": - description: Flag state restored to the baseline. - "404": - description: Not implemented; the TCK falls back to `/start`. - "501": - description: Not implemented; the TCK falls back to `/start`. - - /healthz: - get: - tags: [health] - operationId: health - summary: "[OPTIONAL] Readiness of the control API" - description: | - Reports whether the control API is ready to accept commands. - - **Fallback when not implemented.** Readiness defaults to "the control - port accepts a TCP connection", which the TCK establishes with a - Testcontainers listening-port wait strategy before the first scenario. A - `404` here is therefore not a failure, and the reference implementation - does not serve this path. - - Note this reports the health of the **control API**, not of the backend. - The backend is deliberately unhealthy during outage scenarios while the - control API must stay reachable — otherwise the TCK could not end the - outage. - responses: - "200": - description: Control API ready. - content: - application/json: - schema: - $ref: "#/components/schemas/Health" - "404": - description: Not implemented; readiness falls back to a TCP port check. - "503": - description: Control API not ready yet. - -components: - schemas: - - Health: - type: object - properties: - status: - type: string - enum: [ok] - description: Present and equal to `ok` when the control API is ready. - required: [status] - - Error: - type: object - properties: - message: - type: string - description: Human-readable explanation. Never interpreted by the TCK. - required: [message] diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/errors.feature b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/errors.feature deleted file mode 100644 index 0346df3d..00000000 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/errors.feature +++ /dev/null @@ -1,80 +0,0 @@ -Feature: Provider error handling - - # Every scenario here asserts the same three-part contract, because all three parts matter and - # providers routinely get one of them wrong: - # - # 1. the code default is returned — an application must keep working, - # 2. the correct error code is reported — an application must be able to tell what went wrong, - # 3. nothing is thrown — an unhandled exception from a flag evaluation is never acceptable. - # - # Requires the backend to be seeded with the canonical flag set — see flags/canonical-flags.json. - - Background: - Given a stable provider - - Scenario Outline: Requesting the wrong type returns the code default - # The full non-numeric mismatch matrix. Numeric coercion is a separate question and is covered - # by the @strict-numeric-typing scenarios below, because "is 0.5 an integer?" has a defensible - # wrong answer whereas "is a string a boolean?" does not. - Given a -flag with key "" and a default value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "ERROR" - And the error-code should be "TYPE_MISMATCH" - And no exception should have been thrown - - Examples: a string flag requested as something else - | key | requested | default | - | string-flag | Boolean | false | - | string-flag | Integer | 1 | - | string-flag | Float | 0.1 | - | wrong-flag | Boolean | false | - - Examples: a boolean flag requested as something else - | key | requested | default | - | boolean-flag | String | fallback | - | boolean-flag | Integer | 1 | - | boolean-flag | Float | 0.1 | - - Examples: a numeric flag requested as a non-numeric type - | key | requested | default | - | integer-flag | Boolean | false | - | integer-flag | String | fallback | - | float-flag | Boolean | false | - | float-flag | String | fallback | - - @object - Scenario Outline: Requesting a structured flag as a scalar returns the code default - Given a -flag with key "object-flag" and a default value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "ERROR" - And the error-code should be "TYPE_MISMATCH" - And no exception should have been thrown - - Examples: - | requested | default | - | Boolean | false | - | String | fallback | - | Integer | 1 | - | Float | 0.1 | - - @strict-numeric-typing - Scenario: A float flag is not silently narrowed to an integer - # 'float-flag' resolves to 0.5. Narrowing that to an integer would lose information - # silently, so it must be reported as a type mismatch rather than rounded. - Given a Integer-flag with key "float-flag" and a default value "1" - When the flag was evaluated with details - Then the resolved details value should be "1" - And the reason should be "ERROR" - And the error-code should be "TYPE_MISMATCH" - And no exception should have been thrown - - Scenario: An unknown flag key returns the code default - # 'missing-flag' is deliberately absent from the canonical flag set. - Given a String-flag with key "missing-flag" and a default value "fallback" - When the flag was evaluated with details - Then the resolved details value should be "fallback" - And the reason should be "ERROR" - And the error-code should be "FLAG_NOT_FOUND" - And no exception should have been thrown diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/evaluation.feature b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/evaluation.feature deleted file mode 100644 index e89f174a..00000000 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/evaluation.feature +++ /dev/null @@ -1,59 +0,0 @@ -Feature: Provider flag evaluation - - # Verifies that a provider maps backend responses onto typed resolution details correctly. - # - # This does NOT test the backend's evaluation logic. Every flag in the canonical set resolves - # to its default variant with no targeting involved, so what is under test is purely the - # provider's mapping of a backend response to a value, a variant and a reason. - # - # Requires the backend to be seeded with the canonical flag set — see flags/canonical-flags.json. - - Background: - Given a stable provider - - Scenario Outline: Resolve values with variant and reason - Given a -flag with key "" and a default value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the variant should be "" - And the reason should be "" - And the error-code should be "" - And no exception should have been thrown - - Examples: - | key | type | default | value | variant | reason | - | boolean-flag | Boolean | false | true | on | STATIC | - | string-flag | String | bye | hi | greeting | STATIC | - | integer-flag | Integer | 1 | 10 | ten | STATIC | - | float-flag | Float | 0.1 | 0.5 | half | STATIC | - - Scenario: An integer flag resolves as an integer - # Paired with the float scenario below and with the narrowing scenario in errors.feature. - # Together they pin down that the two numeric types stay distinct rather than both being - # funnelled through one numeric representation. - Given a Integer-flag with key "integer-flag" and a default value "1" - When the flag was evaluated with details - Then the resolved details value should be "10" - And the error-code should be "" - And no exception should have been thrown - - Scenario: A float flag resolves as a float - Given a Float-flag with key "float-flag" and a default value "0.1" - When the flag was evaluated with details - Then the resolved details value should be "0.5" - And the error-code should be "" - And no exception should have been thrown - - @object - Scenario: Resolve a structured value - Given a Object-flag with key "object-flag" and a default value "{}" - When the flag was evaluated with details - Then the variant should be "template" - And the reason should be "STATIC" - And the error-code should be "" - And no exception should have been thrown - And the resolved object value should contain - | key | type | value | - | showImages | Boolean | true | - | title | String | Check out these pics! | - | imagesPerPage | Integer | 100 | diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/events.feature b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/events.feature deleted file mode 100644 index 00e7e5ef..00000000 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/events.feature +++ /dev/null @@ -1,42 +0,0 @@ -@events -Feature: Provider events - - # Verifies that a provider notices changes in its backend and both signals them and acts on - # them. Signalling alone is not enough: a configuration-change event that is not followed by - # a changed evaluation result is a lie, so each scenario asserts the event AND the behaviour. - # - # Outages here are simulated inside the running stack via the control API. No container is - # ever stopped or restarted — see the invariant in openapi/control-api.yaml. - - Background: - Given a stable provider - - @configuration-change - Scenario: A configuration change is signalled and applied - Given a String-flag with key "changing-flag" and a default value "unset" - And a change event handler - When the flag was evaluated with details - And the resolved value is remembered - And the flag was modified - Then the change event handler should have been executed - And the flag should be part of the event payload - When the flag was evaluated with details - Then the resolved details value should have changed - And no exception should have been thrown - - @stale - Scenario: Losing the backend makes the provider stale, regaining it makes it ready again - Given a ready event handler - And a stale event handler - When a ready event was fired - And the connection is lost - Then the stale event handler should have been executed - And the client should be in stale state - When the connection is restored - Then the ready event handler should have been executed - And the client should be in ready state - - # Deliberately NOT covered here: whether a stale provider keeps serving last-known values - # during the outage. That is caching behaviour, which depends on whether the provider holds a - # local copy of the ruleset, and it belongs behind the @caching capability once those - # scenarios are written. See the "Known gaps" section of the README. diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/lifecycle.feature b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/lifecycle.feature deleted file mode 100644 index 25616410..00000000 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/features/lifecycle.feature +++ /dev/null @@ -1,33 +0,0 @@ -@events -Feature: Provider lifecycle - - # Verifies the two terminal outcomes of provider initialisation: reaching READY against a - # healthy backend, and settling into ERROR against one that cannot be reached. - # - # The failure case matters more than it looks. A provider that blocks forever, or throws out - # of provider registration, takes the host application down with it — so the requirement is - # not merely that initialisation fails, but that it fails observably and promptly. - - Scenario: A provider reaching its backend becomes ready - Given a stable provider - And a ready event handler - Then the ready event handler should have been executed - And the client should be in ready state - - @unavailable - Scenario: A provider that cannot reach its backend reports an error - Given a unavailable provider - And a error event handler - Then the error event handler should have been executed within 10000ms - And the client should be in error state - - @unavailable - Scenario: A provider that cannot reach its backend still returns code defaults - Given a unavailable provider - And a error event handler - And a Boolean-flag with key "boolean-flag" and a default value "false" - Then the error event handler should have been executed within 10000ms - When the flag was evaluated with details - Then the resolved details value should be "false" - And the reason should be "ERROR" - And no exception should have been thrown diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/flag_data/canonical-flags.json b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/flag_data/canonical-flags.json deleted file mode 100644 index 343b3ae5..00000000 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/flag_data/canonical-flags.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "$comment": [ - "The canonical flag set the TCK's feature files assume. A backend under test MUST serve an", - "equivalent set under the configuration named 'default'.", - "", - "Expressed in the flagd flag-definition format because that is the only widely implemented", - "vendor-neutral format today. The format is not what matters — the keys, types, variant", - "names and resolved values are. Seed them however your backend seeds flags.", - "", - "Two things are load-bearing and easy to get wrong:", - " * 'missing-flag' MUST NOT exist. Its absence is what the FLAG_NOT_FOUND scenario tests.", - " * No flag here has targeting rules. Every scenario expects reason STATIC, because the TCK", - " tests the provider's mapping of a response, not the backend's evaluation logic." - ], - "flags": { - "boolean-flag": { - "state": "ENABLED", - "variants": { - "on": true, - "off": false - }, - "defaultVariant": "on" - }, - "string-flag": { - "state": "ENABLED", - "variants": { - "greeting": "hi", - "parting": "bye" - }, - "defaultVariant": "greeting" - }, - "integer-flag": { - "state": "ENABLED", - "variants": { - "one": 1, - "ten": 10 - }, - "defaultVariant": "ten" - }, - "float-flag": { - "state": "ENABLED", - "variants": { - "tenth": 0.1, - "half": 0.5 - }, - "defaultVariant": "half" - }, - "object-flag": { - "state": "ENABLED", - "variants": { - "empty": {}, - "template": { - "showImages": true, - "title": "Check out these pics!", - "imagesPerPage": 100 - } - }, - "defaultVariant": "template" - }, - "wrong-flag": { - "$comment": "A string flag, evaluated as a boolean by the TYPE_MISMATCH scenario.", - "state": "ENABLED", - "variants": { - "one": "uno", - "two": "dos" - }, - "defaultVariant": "one" - }, - "changing-flag": { - "$comment": [ - "The flag POST /change mutates. The TCK asserts only that its resolved value differs", - "after the change, so which of the two variants you start from does not matter." - ], - "state": "ENABLED", - "variants": { - "foo": "foo", - "bar": "bar" - }, - "defaultVariant": "foo" - } - } -} From d6de5dc90478ea87950a37a6be89bc0d1f150d11 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 14:40:15 +0200 Subject: [PATCH 7/7] feat(provider-tck): add the @lifecycle capability lifecycle.feature was gated by @events, which was wrong in both directions. An SDK dispatches PROVIDER_READY around initialize for any provider (openfeature/provider/_registry.py), so a provider that declares @events passes the readiness scenario without demonstrating anything -- a NoOpProvider passes it identically. The gate made the scenario vacuous for exactly the providers it admitted. Conversely a stateless provider such as OFREP has a real initialisation to verify but no event stream of its own to declare @events for, so the gate shut it out of a scenario it should be held to. The spec revision pinned by the submodule retags the feature to @lifecycle and adds the capability to Appendix F. Add the matching enum member; plugin.py registers the marker by iterating the enum, so nothing else changes. Neither in-memory self-test declares it. They have no backend to reach, so their readiness scenario was passing vacuously too, and a skip with a reason is the honest outcome. 54 passed, 9 skipped, 2 xfailed. Signed-off-by: Simon Schrottner --- tools/openfeature-provider-tck/README.md | 14 ++++++++++++- .../contrib/tools/provider_tck/capability.py | 21 +++++++++++++++++++ .../tests/test_controllable_conformance.py | 5 +++++ .../tests/test_in_memory_conformance.py | 8 +++++++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/tools/openfeature-provider-tck/README.md b/tools/openfeature-provider-tck/README.md index af37eda1..52736040 100644 --- a/tools/openfeature-provider-tck/README.md +++ b/tools/openfeature-provider-tck/README.md @@ -87,6 +87,7 @@ SKIPPED provider does not declare capability @stale. | Capability | Tag | Meaning | | --- | --- | --- | +| `Capability.LIFECYCLE` | `@lifecycle` | reaches its backend during initialisation, observably and promptly | | `Capability.EVENTS` | `@events` | emits lifecycle events at all | | `Capability.STALE` | `@stale` | enters `STALE` and emits `PROVIDER_STALE` on backend loss | | `Capability.CONFIGURATION_CHANGE` | `@configuration-change` | detects configuration changes and emits `PROVIDER_CONFIGURATION_CHANGED` | @@ -96,6 +97,13 @@ SKIPPED provider does not declare capability @stale. | `Capability.TARGETING` | `@targeting` | reserved; no scenarios yet | | `Capability.CACHING` | `@caching` | reserved; no scenarios yet | +`@lifecycle` and `@events` are deliberately separate, and the split matters in both directions. An +SDK dispatches `PROVIDER_READY` around `initialize` for *any* provider, so a provider declaring only +`@events` passes the readiness scenario without demonstrating anything — a `NoOpProvider` passes it +identically. Meanwhile a stateless provider has a real initialisation to verify but no event stream +of its own to declare `@events` for, and gating on `@events` shut it out of a scenario it should be +held to. + Untagged scenarios are mandatory and always run. `capabilities` defaults to everything — narrow it rather than widening it: start from the default, run the suite, and remove only what your provider genuinely cannot do. @@ -205,11 +213,15 @@ This mirrors what `openfeature-flagd-api-testkit` already does for the flagd tes | `test_in_process_control` | `InProcessControl` | pins what the Gherkin cannot assert about itself | ``` -56 passed, 7 skipped, 2 xfailed +54 passed, 9 skipped, 2 xfailed ``` No Docker, no network, under a second. +Neither in-memory suite declares `@lifecycle`, so the three lifecycle scenarios are skipped in both. +That is the point: with no backend to reach, they would pass without testing anything — which is +what they did while the feature was gated on `@events`. + ## Known gaps - **Evaluation context passthrough is unverifiable.** The scenarios build evaluation contexts but diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py index 04490864..0444352b 100644 --- a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py @@ -25,6 +25,27 @@ class Capability(str, Enum): Scenarios with no capability tag are mandatory and always run. """ + LIFECYCLE = "lifecycle" + """Provider reaches its backend during initialisation, observably and promptly. + + Deliberately separate from :attr:`EVENTS`, because the two are independent in + both directions. + + An SDK dispatches ``PROVIDER_READY`` around ``initialize`` for *any* + provider, so a provider that declares ``EVENTS`` passes the readiness + scenario without demonstrating anything -- a ``NoOpProvider`` passes it + identically. Gating on ``EVENTS`` therefore made the scenario vacuous for + exactly the providers that declared it. + + Conversely a stateless provider -- one that resolves every flag with a fresh + request and holds nothing between them -- has a real initialisation to + verify while having no event stream of its own to declare ``EVENTS`` for. + Gating on ``EVENTS`` shut it out of a scenario it should be held to. + + Declare it if initialisation actually contacts the backend and its outcome, + success or failure, is observable to the application. + """ + EVENTS = "events" """Provider emits lifecycle events at all, at minimum ``PROVIDER_READY``.""" diff --git a/tools/openfeature-provider-tck/tests/test_controllable_conformance.py b/tools/openfeature-provider-tck/tests/test_controllable_conformance.py index 3ebd240e..f127243c 100644 --- a/tools/openfeature-provider-tck/tests/test_controllable_conformance.py +++ b/tools/openfeature-provider-tck/tests/test_controllable_conformance.py @@ -33,6 +33,11 @@ def tck_config() -> TckConfig: connection to lose, and ``InProcessControl`` does not implement ``ConnectionControl``. ``CONFIGURATION_CHANGE`` is what this suite adds over the plain in-memory one, and it is the whole point of it. + + ``LIFECYCLE`` stays undeclared for the same reason as in + ``test_in_memory_conformance``: there is no backend to reach during + initialisation, so the readiness scenario would pass here without testing + anything. It did exactly that while the feature was gated on ``@events``. """ control = InProcessControl() return TckConfig( diff --git a/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py index de025022..648ba05a 100644 --- a/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py +++ b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py @@ -85,6 +85,14 @@ def tck_config() -> TckConfig: operation the control cannot perform. * ``TARGETING`` and ``CACHING`` -- omitted because no scenario carries their tags yet, so leaving them out skips nothing. + * ``LIFECYCLE`` -- omitted because there is no backend to reach. The + capability asserts that initialisation actually contacts a backend and + that the outcome is observable; this provider's ``initialize`` is a no-op + and the SDK dispatches ``PROVIDER_READY`` around it regardless, so the + readiness scenario would pass here without testing anything. It passed + vacuously while the feature was gated on ``EVENTS``, which is precisely + the failure mode the split of ``@lifecycle`` from ``@events`` exists to + end. A skip with a reason is the honest outcome. """ return TckConfig( name="in-memory",