diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8c09e514..d80adb19 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,6 +66,9 @@ jobs: tools/openfeature-flagd-api-testkit: - 'tools/openfeature-flagd-api-testkit/**' - 'uv.lock' + tools/openfeature-provider-tck: + - 'tools/openfeature-provider-tck/**' + - 'uv.lock' build: needs: changes diff --git a/.gitmodules b/.gitmodules index 7e8bf9ed..31678c42 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "providers/openfeature-provider-flagd/openfeature/test-harness"] path = providers/openfeature-provider-flagd/openfeature/test-harness url = https://github.com/open-feature/flagd-testbed.git +[submodule "tools/openfeature-provider-tck/spec"] + path = tools/openfeature-provider-tck/spec + url = https://github.com/open-feature/spec diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 94d58394..6bb08620 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -8,5 +8,6 @@ "providers/openfeature-provider-unleash": "0.1.2", "tools/openfeature-flagd-api": "1.0.0", "tools/openfeature-flagd-core": "1.0.0", - "tools/openfeature-flagd-api-testkit": "0.1.0" + "tools/openfeature-flagd-api-testkit": "0.1.0", + "tools/openfeature-provider-tck": "0.1.0" } diff --git a/pyproject.toml b/pyproject.toml index 647571bc..e250a4b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "openfeature-flagd-api", "openfeature-flagd-core", "openfeature-flagd-api-testkit", + "openfeature-provider-tck", ] [dependency-groups] @@ -43,6 +44,7 @@ openfeature-provider-unleash = { workspace = true } openfeature-flagd-api = { workspace = true } openfeature-flagd-core = { workspace = true } openfeature-flagd-api-testkit = { workspace = true } +openfeature-provider-tck = { workspace = true } [tool.uv.workspace] members = [ @@ -58,7 +60,10 @@ exclude = [ ".venv", "__pycache__", "venv", - "providers/openfeature-provider-flagd/src/openfeature/schemas/**" + "providers/openfeature-provider-flagd/src/openfeature/schemas/**", + # Submodules of other repositories: not ours to lint or format. + "providers/openfeature-provider-flagd/openfeature/spec/**", + "tools/openfeature-provider-tck/spec/**", ] [tool.ruff.lint] diff --git a/release-please-config.json b/release-please-config.json index a6335415..29cfce44 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -99,6 +99,15 @@ "extra-files": [ "README.md" ] + }, + "tools/openfeature-provider-tck": { + "package-name": "openfeature-provider-tck", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true, + "versioning": "default", + "extra-files": [ + "README.md" + ] } }, "changelog-sections": [ diff --git a/tools/openfeature-provider-tck/.gitignore b/tools/openfeature-provider-tck/.gitignore new file mode 100644 index 00000000..06664622 --- /dev/null +++ b/tools/openfeature-provider-tck/.gitignore @@ -0,0 +1,7 @@ +# Copied from the open-feature/spec submodule by hatch_build_sync.py. +# DO NOT EDIT the copies, and do not commit them: the canonical definitions live +# in spec/specification/assets/provider-tck/, and the revision this package is +# built against is recorded by the submodule pin. +src/openfeature/contrib/tools/provider_tck/features/ +src/openfeature/contrib/tools/provider_tck/flag_data/ +src/openfeature/contrib/tools/provider_tck/control-api.yaml diff --git a/tools/openfeature-provider-tck/LICENSE b/tools/openfeature-provider-tck/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/tools/openfeature-provider-tck/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/openfeature-provider-tck/README.md b/tools/openfeature-provider-tck/README.md new file mode 100644 index 00000000..52736040 --- /dev/null +++ b/tools/openfeature-provider-tck/README.md @@ -0,0 +1,235 @@ +# OpenFeature Provider TCK (Python) + +A conformance suite any OpenFeature Python provider can adopt to verify that it implements the +provider contract of the specification. + +OpenFeature's central promise is that swapping providers does not change application behaviour. +Nothing verifies that today, and every provider tests differently — so "implements the provider +contract" is an unverified claim, and a behavioural difference between two providers is discovered +by the application that trips over it. + +This package is the Python implementation of [Appendix F][appendix-f]. It runs the same Gherkin +scenarios, against the same canonical flag set, driven through the same backend control API, as +every other language's TCK. That shared basis is the point: "conformant" only means something if the +question is identical everywhere. + +Tracking issue: [open-feature/spec#417][tracking]. + +## Status + +**Proof of concept.** The scenario set is a representative subset covering each architectural +mechanism once, not exhaustive coverage. Breaking changes should be expected. + +## Adopting it + +One fixture and one call. It uses **pytest-bdd**, the same runner the flagd provider and the flagd +testkit already use, so an adopting package gains no new test framework. + +```python +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + TckConfig, + features_path, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = MyBackendControl() + return TckConfig( + name="my-provider", + control=control, + new_provider=lambda: MyProvider(control.address), + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + + +scenarios(features_path()) +``` + +There is **no `conftest.py` to write and nothing to import for the steps**. The step definitions +arrive through this package's pytest plugin, registered via a `pytest11` entry point, so installing +the package is all it takes. + +The TCK owns the whole lifecycle: registering the provider under a suite-scoped domain, awaiting +events, resetting the backend between scenarios, releasing it at the end. **If you find yourself +writing test infrastructure, that is a defect here rather than something for you to work around.** + +pytest-bdd generates one test per scenario — and one per row of a Scenario Outline — so failures +name a scenario and `-k` selects one as usual. The feature files and canonical flag set are packaged +inside the distribution, so **adopting this package needs no git submodule** — see +[Where the assets come from](#where-the-assets-come-from). + +### Timings + +`TckConfig.event_timeout` is the knob that matters. Providers observe backend changes on wildly +different timescales — a streaming provider sees a configuration change in milliseconds, one that +polls every 30 seconds may need most of a poll interval. Set it to comfortably exceed your +worst-case detection latency, or the suite reports timeouts that are really just impatience. + +## Capabilities + +Not every provider implements every optional part of the contract. Each scenario exercising an +optional part carries a Gherkin tag, pytest-bdd turns that tag into a pytest marker, and a provider +declares what it supports. + +**A scenario whose capability was not declared is reported as skipped, with the reason — never as +passed.** A conformance suite that quietly goes green on scenarios it did not run is worse than no +suite at all, so `pytest.skip` carries the reason into the report: + +``` +SKIPPED provider does not declare capability @stale. + Declared: @events @object @strict-numeric-typing +``` + +| Capability | Tag | Meaning | +| --- | --- | --- | +| `Capability.LIFECYCLE` | `@lifecycle` | reaches its backend during initialisation, observably and promptly | +| `Capability.EVENTS` | `@events` | emits lifecycle events at all | +| `Capability.STALE` | `@stale` | enters `STALE` and emits `PROVIDER_STALE` on backend loss | +| `Capability.CONFIGURATION_CHANGE` | `@configuration-change` | detects configuration changes and emits `PROVIDER_CONFIGURATION_CHANGED` | +| `Capability.OBJECT` | `@object` | supports structured flag values | +| `Capability.UNAVAILABLE_INIT` | `@unavailable` | reports an error state instead of hanging against a dead backend | +| `Capability.STRICT_NUMERIC_TYPING` | `@strict-numeric-typing` | does not coerce between integer and float | +| `Capability.TARGETING` | `@targeting` | reserved; no scenarios yet | +| `Capability.CACHING` | `@caching` | reserved; no scenarios yet | + +`@lifecycle` and `@events` are deliberately separate, and the split matters in both directions. An +SDK dispatches `PROVIDER_READY` around `initialize` for *any* provider, so a provider declaring only +`@events` passes the readiness scenario without demonstrating anything — a `NoOpProvider` passes it +identically. Meanwhile a stateless provider has a real initialisation to verify but no event stream +of its own to declare `@events` for, and gating on `@events` shut it out of a scenario it should be +held to. + +Untagged scenarios are mandatory and always run. `capabilities` defaults to everything — narrow it +rather than widening it: start from the default, run the suite, and remove only what your provider +genuinely cannot do. + +`@strict-numeric-typing` deserves a note, because unlike the others it is **not** an optional +feature. The specification requires `TYPE_MISMATCH` when the requested type cannot be satisfied, and +narrowing `0.5` to `0` loses information silently. It is a capability only so a provider with the +defect can adopt today and see the gap reported explicitly rather than being unable to adopt at all. +Not declaring it is an admission of a known bug. + +## Controlling the backend + +`BackendControl` is the single seam between the scenarios and whatever manipulates the backend. Step +definitions never talk to a backend directly, which is why the same Gherkin runs unchanged against a +containerised backend and against a provider manipulated in-process. + +**If your provider talks to a backend, drive it over the HTTP control API** — the document is +available as `control_api_spec()`. That API is the normative contract for those providers, and it is +what makes a conformance claim portable: another language's TCK drives the same endpoints against +the same stack and must get the same answers. + +Two of its requirements are easy to get wrong: + +- **Containers are never stopped or restarted mid-suite.** Unavailability is simulated *inside* the + running stack. Container orchestrators assign host ports dynamically and cannot reliably preserve + them across a restart, so restarting silently invalidates every provider already pointed at the + old port, and the failure looks like a flaky provider. +- **`/start` resets flag state; `/restart` preserves it.** An outage must be observable as a change + in availability, never as a change in flag values. + +### Providers with no backend + +An in-memory, environment-variable or file-based provider has nothing to connect to. Those may +control the backend in-process, where flag operations are direct manipulations of the provider's own +state. `InProcessControl` is the reference. + +This is a narrow allowance and the obvious thing to abuse. **A provider with an external backend +must use the control API.** Reaching into an external backend from inside the test process — a +test-only admin client, a shared database handle, a hook inside the provider — produces a suite that +passes while proving nothing, because the path it exercised is not the path the contract describes. + +Connection-dependent scenarios have no meaning without a connection, so a backend-less control +simply does not implement `ConnectionControl`, leaves `STALE` and `UNAVAILABLE_INIT` undeclared, and +those scenarios are skipped with their reason. + +## Findings + +Two, both confirmed by running the suite rather than by reading code. + +### 1. A boolean satisfies an Integer request + +`boolean-flag` evaluated through `get_integer_details` returns `True` with reason `STATIC` and **no +error code**, where the specification requires the code default and `TYPE_MISMATCH`. The client +type-checks with `isinstance(value, int)`, and `bool` is a subclass of `int` in Python. + +This is **Python-specific** — the identical scenario passes in every other language's suite, which +is a fair advertisement for having more than one implementation. Tracked as +[open-feature/python-sdk#619](https://github.com/open-feature/python-sdk/issues/619). + +The self-test marks that one row `xfail(strict=True)` with a pointer to the issue, so it stays +visible in the report and un-hides itself automatically once the SDK is fixed. + +### 2. The in-memory provider cannot update its flag set + +[Appendix A][appendix-a] requires an SDK's in-memory provider to support updating the flag set and +emitting `PROVIDER_CONFIGURATION_CHANGED`. Python's copies its mapping in the constructor and +exposes nothing to change it. Tracked as +[open-feature/python-sdk#620](https://github.com/open-feature/python-sdk/issues/620). + +Only half the machinery is missing — `AbstractProvider` already supplies +`emit_provider_configuration_changed` — which is why `ControllableInMemoryProvider` here is a small +subclass rather than a reimplementation, and why it should port back to the SDK as a method. + +## Where the assets come from + +The Gherkin feature files, the canonical flag set and the control-API document are **not owned by +this repository**. They are the language-agnostic conformance artifacts defined in +[open-feature/spec][spec] under `specification/assets/provider-tck/`, and every language's TCK ships +the same ones — which is the only reason a conformance claim means the same thing in Python as it +does in Java. + +**Adopting this package needs no submodule.** The assets are copied into the wheel and the sdist at +build time, so `pip install openfeature-provider-tck` gives you everything the suite runs on. + +**Contributing to this package does.** The spec is a git submodule at +`tools/openfeature-provider-tck/spec`, and the copies under +`src/openfeature/contrib/tools/provider_tck/` are gitignored and generated: + +```bash +git submodule update --init tools/openfeature-provider-tck/spec +poe test # runs `poe sync-spec-assets` first +``` + +The copies carry a `DO-NOT-EDIT.txt` because editing them forks the definition of conformance, which +is the one thing this suite exists to prevent. A change goes to [open-feature/spec][spec] first; +then bump the submodule pin here. Committing no copies means the spec revision this package targets +is recorded by the pin and nowhere else, so the two cannot drift apart unnoticed. + +This mirrors what `openfeature-flagd-api-testkit` already does for the flagd test harness. + +## The self-tests + +| Suite | Subject | Why | +| --- | --- | --- | +| `test_in_memory_conformance` | the SDK's `InMemoryProvider` | reference adoption for a backend-less provider | +| `test_controllable_conformance` | `ControllableInMemoryProvider` | the only suite that exercises the configuration-change path — see finding 2 | +| `test_in_process_control` | `InProcessControl` | pins what the Gherkin cannot assert about itself | + +``` +54 passed, 9 skipped, 2 xfailed +``` + +No Docker, no network, under a second. + +Neither in-memory suite declares `@lifecycle`, so the three lifecycle scenarios are skipped in both. +That is the point: with no backend to reach, they would pass without testing anything — which is +what they did while the feature was gated on `@events`. + +## Known gaps + +- **Evaluation context passthrough is unverifiable.** The scenarios build evaluation contexts but + cannot assert one *reached* the backend. That needs an echo operation on the control API. +- **No HTTP control client yet.** It arrives with the first containerised adopter. +- **Caching, hooks and flag metadata** are not covered. + +[appendix-a]: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md +[appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md +[spec]: https://github.com/open-feature/spec +[tracking]: https://github.com/open-feature/spec/issues/417 diff --git a/tools/openfeature-provider-tck/hatch_build.py b/tools/openfeature-provider-tck/hatch_build.py new file mode 100644 index 00000000..4b6f1d84 --- /dev/null +++ b/tools/openfeature-provider-tck/hatch_build.py @@ -0,0 +1,51 @@ +"""Hatch build hook to copy the canonical conformance assets into the package. + +The feature files, the canonical flag set and the control-API document are owned +by open-feature/spec and reach this package through a git submodule, so nothing +in this repository can fork the definition of conformance. They are copied into +the source tree at build time and force-included into the distribution, which is +what lets an *adopter* install the wheel and run the suite with no submodule of +their own. +""" + +import sys +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +# Hatchling loads this file by path rather than importing it as part of a +# package, so its directory is not on sys.path and the sibling sync module -- +# the single definition of what gets copied where -- would not be importable. +sys.path.insert(0, str(Path(__file__).parent)) + +from hatch_build_sync import FILES, PACKAGE_REL, SPEC_ASSETS, TREES, sync + + +class SpecAssetsCopyHook(BuildHookInterface): + PLUGIN_NAME = "spec-assets-copy" + + def initialize(self, version: str, build_data: dict) -> None: + root = Path(self.root) + copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + + # Building from a checkout: refresh from the submodule, so what ships is + # always the revision the pin names. Building from an sdist: there is no + # submodule, but the copies are already in the tree. + if SPEC_ASSETS.exists(): + sync() + elif not all(path.exists() for path in copies): + missing = ", ".join(str(p) for p in copies if not p.exists()) + msg = ( + f"Conformance assets missing ({missing}) and the open-feature/spec " + f"submodule is not checked out at {SPEC_ASSETS}. Run " + "`git submodule update --init tools/openfeature-provider-tck/spec`." + ) + raise FileNotFoundError(msg) + + # Force-include the gitignored copies into both sdist and wheel. + force = build_data.setdefault("force_include", {}) + for path in copies: + for member in [path] if path.is_file() else path.rglob("*"): + if member.is_file(): + rel = str(member.relative_to(root)) + force[rel] = rel diff --git a/tools/openfeature-provider-tck/hatch_build_sync.py b/tools/openfeature-provider-tck/hatch_build_sync.py new file mode 100644 index 00000000..f31bc55b --- /dev/null +++ b/tools/openfeature-provider-tck/hatch_build_sync.py @@ -0,0 +1,56 @@ +"""Copy the canonical conformance assets from the spec submodule into the package. + +Used by `poe sync-spec-assets` for local development and CI testing. The hatch +build hook (hatch_build.py) handles inclusion in the wheel and sdist. + +The assets are owned by open-feature/spec, not by this repository. Copying them +in at build time -- rather than committing copies -- means the spec revision this +package was built against is recorded by the submodule pin and nowhere else, so +the two cannot drift apart unnoticed. An *adopter* installing the wheel still +needs no submodule: the copies are inside the distribution. +""" + +import shutil +from pathlib import Path + +ROOT = Path(__file__).parent +SPEC_ASSETS = (ROOT / "spec/specification/assets/provider-tck").resolve() +PACKAGE_REL = Path("src/openfeature/contrib/tools/provider_tck") +DEST_BASE = ROOT / PACKAGE_REL + +DO_NOT_EDIT = ( + "Generated by hatch_build_sync.py from the open-feature/spec submodule.\n" + "DO NOT EDIT. Changes belong in open-feature/spec under\n" + "specification/assets/provider-tck/, then bump the submodule pin.\n" +) + +# (source directory or file, destination) relative to SPEC_ASSETS / DEST_BASE. +TREES = [("gherkin", "features"), ("flags", "flag_data")] +FILES = [("openapi/control-api.yaml", "control-api.yaml")] + + +def sync() -> None: + if not SPEC_ASSETS.exists(): + msg = ( + f"Conformance assets not found at {SPEC_ASSETS}. " + "Make sure submodules are initialized: " + "`git submodule update --init tools/openfeature-provider-tck/spec`." + ) + raise FileNotFoundError(msg) + + for src_name, dest_name in TREES: + dest = DEST_BASE / dest_name + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(SPEC_ASSETS / src_name, dest) + (dest / "DO-NOT-EDIT.txt").write_text(DO_NOT_EDIT, encoding="utf-8") + + for src_name, dest_name in FILES: + dest = DEST_BASE / dest_name + if dest.exists(): + dest.unlink() + shutil.copy2(SPEC_ASSETS / src_name, dest) + + +if __name__ == "__main__": + sync() diff --git a/tools/openfeature-provider-tck/pyproject.toml b/tools/openfeature-provider-tck/pyproject.toml new file mode 100644 index 00000000..ff0cbe43 --- /dev/null +++ b/tools/openfeature-provider-tck/pyproject.toml @@ -0,0 +1,87 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "openfeature-provider-tck" +version = "0.1.0" +description = "OpenFeature provider conformance suite (TCK)" +readme = "README.md" +authors = [{ name = "OpenFeature", email = "openfeature-core@groups.io" }] +license = { file = "LICENSE" } +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Framework :: Pytest", +] +keywords = ["openfeature", "conformance", "tck", "feature-flags"] +dependencies = [ + "openfeature-sdk>=0.8.2", + "pytest>=8.4.0", + # Same runner the flagd provider and the flagd testkit already use, so an + # adopting module gains no new test framework. + "pytest-bdd>=8.1.0,<9.0.0", +] +requires-python = ">=3.10" + +[project.urls] +Homepage = "https://github.com/open-feature/python-sdk-contrib" + +# Shipping the step definitions as a pytest plugin is what keeps adoption to a +# single fixture: pytest-bdd resolves steps through the fixture system, and +# fixtures from an installed plugin are visible to every test, so an adopter +# never has to `from ... import *` to pull the vocabulary in. +[project.entry-points.pytest11] +openfeature_provider_tck = "openfeature.contrib.tools.provider_tck.plugin" + +[dependency-groups] +dev = [ + "coverage[toml]>=7.10.0,<8.0.0", + "mypy>=1.18.0,<2.0.0", + "poethepoet>=0.37.0", +] + +[tool.hatch.build.targets.sdist] +# The conformance assets are gitignored copies of the spec submodule; the build +# hook force-includes them so an sdist builds into a wheel without a submodule. +force-include = {} +# Which is why the submodule itself has no business in the sdist: it is the whole +# spec repository, and only the copies of the four assets are needed downstream. +exclude = ["/spec"] + +[tool.hatch.build.targets.wheel] +packages = ["src/openfeature"] +# Ship the conformance assets even though they are gitignored: an adopter +# installing this package must need no submodule of their own. +artifacts = [ + "src/openfeature/contrib/tools/provider_tck/features/", + "src/openfeature/contrib/tools/provider_tck/flag_data/", + "src/openfeature/contrib/tools/provider_tck/control-api.yaml", +] + +[tool.hatch.build.hooks.custom] + +[tool.mypy] +mypy_path = "src" +files = ["src", "tests"] +python_version = "3.10" +namespace_packages = true +explicit_package_bases = true +local_partial_types = true +allow_redefinition_new = true +fixed_format_cache = true +pretty = true +strict = true +disallow_any_generics = false + +[tool.coverage.run] +omit = ["tests/**"] + +[tool.poe.tasks] +sync-spec-assets = "python hatch_build_sync.py" +test = ["sync-spec-assets", {cmd = "pytest tests"}] +test-cov = ["sync-spec-assets", {cmd = "coverage run -m pytest tests"}] +cov-report = "coverage xml" +cov = ["test-cov", "cov-report"] +mypy = "mypy" diff --git a/tools/openfeature-provider-tck/spec b/tools/openfeature-provider-tck/spec new file mode 160000 index 00000000..dfa16586 --- /dev/null +++ b/tools/openfeature-provider-tck/spec @@ -0,0 +1 @@ +Subproject commit dfa16586d91ca020ef1b3b82a7c972d833ff8f29 diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py new file mode 100644 index 00000000..8b615296 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/__init__.py @@ -0,0 +1,138 @@ +"""The OpenFeature Provider Conformance Suite (TCK) for Python. + +The suite answers one question: does this provider map its backend onto the +OpenFeature provider contract correctly? It is the Python implementation of +`Appendix F`_ of the specification, and it runs the same Gherkin scenarios, +against the same canonical flag set, that every other language's TCK runs. That +shared basis is the whole point -- "conformant" only means something if the +question is identical everywhere. + +**What a provider author writes.** One fixture and one call:: + + import pytest + from pytest_bdd import scenarios + + from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, + features_path, + ) + + @pytest.fixture(scope="session") + def tck_config(): + control = InProcessControl() + return TckConfig( + name="my-provider", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + + scenarios(features_path()) + +``scenarios()`` is pytest-bdd's own, called directly rather than wrapped: it +injects the generated tests into the *calling module* by walking the stack, so a +convenience wrapper around it would deposit them inside this package instead. + +The step definitions arrive through this package's pytest plugin, so there is +nothing to import for them and no ``conftest.py`` to write. Everything else -- +registering the provider, awaiting events, resetting the backend between +scenarios, tearing down -- belongs to the TCK. If you find yourself writing test +infrastructure, that is a defect here rather than something for you to work +around. + +.. _Appendix F: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md +""" + +from __future__ import annotations + +import importlib.resources + +from .capability import ALL_CAPABILITIES, Capability +from .config import TckConfig +from .control import ( + BackendControl, + ConnectionControl, + UnsupportedControlError, +) +from .inprocess import InProcessControl +from .provider import ( + CHANGING_FLAG_KEY, + ControllableInMemoryProvider, + canonical_flag_set, +) + +__all__ = [ + "ALL_CAPABILITIES", + "CHANGING_FLAG_KEY", + "BackendControl", + "Capability", + "ConnectionControl", + "ControllableInMemoryProvider", + "InProcessControl", + "TckConfig", + "UnsupportedControlError", + "canonical_flag_set", + "canonical_flags_json", + "control_api_spec", + "features_path", +] + +# NOTE ON THE SOURCE OF TRUTH +# +# The files under features/ and flag_data/, and control-api.yaml, are NOT owned +# by this repository and are NOT committed to it. They are copies of the +# language-agnostic conformance artifacts defined in open-feature/spec under +# specification/assets/provider-tck/, which reaches this package as a git +# submodule at tools/openfeature-provider-tck/spec and is copied in at build +# time by hatch_build.py. The copies are gitignored, so the only record of which +# spec revision this package targets is the submodule pin, and the two cannot +# drift apart unnoticed. +# +# They are copied into the distribution, so an adopter installing this package +# needs no submodule of their own; only a contributor to this package does. +# +# Changes belong in open-feature/spec first, followed by a bump of the submodule +# pin -- editing the copies locally forks the definition of conformance, which is +# the one thing this suite exists to prevent. +# See https://github.com/open-feature/spec/issues/417. + +_PACKAGE = "openfeature.contrib.tools.provider_tck" + + +def features_path() -> str: + """Return the directory holding the canonical feature files. + + Packaged with this distribution, so a consumer needs no submodule and no + particular directory layout. Hand it to pytest-bdd's ``scenarios()``, which + accepts an absolute path:: + + scenarios(features_path()) + + pytest-bdd generates one test per scenario -- and one per row of a Scenario + Outline -- so failures name a scenario and ``-k`` selects one as usual. + """ + return str(importlib.resources.files(_PACKAGE) / "features") + + +def canonical_flags_json() -> str: + """Return the canonical flag set as raw JSON, in the flagd flag-definition format. + + This is the flag set every scenario assumes, and a backend under test must + serve an equivalent one. The format is not what matters -- the keys, types, + variant names and resolved values are. Seed them however your backend seeds + flags. + + Exposed so an adopting provider can seed a backend from the canonical + definition rather than transcribing it, transcription being the usual way + the two drift apart. + """ + ref = importlib.resources.files(_PACKAGE) / "flag_data" / "canonical-flags.json" + return ref.read_text(encoding="utf-8") + + +def control_api_spec() -> str: + """Return the OpenAPI document a containerised backend under test must implement.""" + ref = importlib.resources.files(_PACKAGE) / "control-api.yaml" + return ref.read_text(encoding="utf-8") diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py new file mode 100644 index 00000000..0444352b --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/capability.py @@ -0,0 +1,112 @@ +"""Optional parts of the provider contract, and the Gherkin tags that gate them.""" + +from __future__ import annotations + +from enum import Enum + +__all__ = ["Capability"] + + +class Capability(str, Enum): + """An optional part of the OpenFeature provider contract. + + Not every provider implements every part of the specification. A provider + backed by a static file has no meaningful notion of going stale; one with no + streaming transport cannot emit configuration-change events. Rather than + forcing such providers to fail scenarios they were never going to satisfy, + each declares what it supports through :attr:`TckConfig.capabilities`. + + Every capability corresponds to exactly one Gherkin tag. pytest-bdd turns + those tags into pytest markers, and a scenario carrying a marker whose + capability was not declared is skipped with the reason reported -- never + passed. A conformance suite that quietly goes green on scenarios it did not + run is worse than no suite at all. + + Scenarios with no capability tag are mandatory and always run. + """ + + LIFECYCLE = "lifecycle" + """Provider reaches its backend during initialisation, observably and promptly. + + Deliberately separate from :attr:`EVENTS`, because the two are independent in + both directions. + + An SDK dispatches ``PROVIDER_READY`` around ``initialize`` for *any* + provider, so a provider that declares ``EVENTS`` passes the readiness + scenario without demonstrating anything -- a ``NoOpProvider`` passes it + identically. Gating on ``EVENTS`` therefore made the scenario vacuous for + exactly the providers that declared it. + + Conversely a stateless provider -- one that resolves every flag with a fresh + request and holds nothing between them -- has a real initialisation to + verify while having no event stream of its own to declare ``EVENTS`` for. + Gating on ``EVENTS`` shut it out of a scenario it should be held to. + + Declare it if initialisation actually contacts the backend and its outcome, + success or failure, is observable to the application. + """ + + EVENTS = "events" + """Provider emits lifecycle events at all, at minimum ``PROVIDER_READY``.""" + + STALE = "stale" + """Provider enters ``STALE`` and emits ``PROVIDER_STALE`` when it loses its backend.""" + + CONFIGURATION_CHANGE = "configuration-change" + """Provider detects configuration changes and emits ``PROVIDER_CONFIGURATION_CHANGED``.""" + + OBJECT = "object" + """Provider supports structured (object) flag values.""" + + UNAVAILABLE_INIT = "unavailable" + """Provider reports an error state promptly against a backend it cannot reach.""" + + STRICT_NUMERIC_TYPING = "strict-numeric-typing" + """Provider keeps the integer and float types distinct instead of coercing between them. + + Unlike every other entry here this is not an optional feature. The + specification requires a provider to report ``TYPE_MISMATCH`` when the + requested type cannot be satisfied, and narrowing ``0.5`` to ``0`` to satisfy + an integer request loses information silently -- the worst failure mode a + feature flag has, because the application sees a plausible value and no + error at all. + + It is a capability only so that a provider with this defect can adopt the + suite today and see the gap reported as an explicit skip, rather than being + unable to adopt at all. Not declaring it is an admission of a known bug, not + a design choice. Declare it as soon as the provider is fixed. + """ + + TARGETING = "targeting" + """Reserved. No scenario carries this tag: targeting is backend evaluation logic.""" + + CACHING = "caching" + """Reserved; no scenario carries this tag yet.""" + + @property + def tag(self) -> str: + """Return the Gherkin tag, with its leading at-sign, that gates this capability.""" + return f"@{self.value}" + + def __str__(self) -> str: + return self.tag + + +ALL_CAPABILITIES: frozenset[Capability] = frozenset(Capability) +"""Every capability the TCK recognises. + +A reasonable starting point for a new adoption: declare everything, run the +suite, and remove only what the provider genuinely cannot do. Narrowing from the +full set surfaces gaps; widening towards it hides them. +""" + +_BY_MARKER: dict[str, Capability] = {c.value: c for c in Capability} + + +def capability_for_marker(name: str) -> Capability | None: + """Map a pytest marker name onto the capability it gates, if any. + + A marker that does not name a capability gates nothing, which is what lets + the canonical feature files carry organisational tags freely. + """ + return _BY_MARKER.get(name) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py new file mode 100644 index 00000000..77b783cd --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/config.py @@ -0,0 +1,179 @@ +"""The contract a provider author implements to run the suite.""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Iterable +from dataclasses import dataclass, field + +from openfeature.provider import FeatureProvider + +from .capability import ALL_CAPABILITIES, Capability +from .control import BackendControl + +__all__ = ["ProviderFactory", "TckConfig"] + +ProviderFactory = Callable[[], FeatureProvider] +"""Creates the provider under test. + +A factory rather than a single instance because each scenario gets its own +provider, and because a provider often cannot be configured before the suite +starts -- a container stack's host ports do not exist until it is up. +""" + +DEFAULT_EVENT_TIMEOUT = 12.0 +DEFAULT_READY_TIMEOUT = 30.0 + + +@dataclass(frozen=True) +class TckConfig: + """Everything the TCK needs to test one provider. + + An adopting module supplies this through a session-scoped ``tck_config`` + fixture; the TCK owns everything else -- registering the provider, awaiting + events, resetting the backend between scenarios, tearing down. If you find + yourself writing test infrastructure, that is a defect in this package + rather than something for you to work around. + """ + + name: str + """Identifies the suite in test output, and scopes the OpenFeature domain + the TCK registers providers under so two suites in the same session do not + observe each other's providers. + + Use something that reads well in a failure message: ``"flagd-rpc"``, + ``"in-memory"``. + """ + + control: BackendControl + """The seam through which the TCK manipulates the backend. + + See :class:`~.control.BackendControl` for which implementation is right for + your provider. The short version: a provider with a real backend drives it + over the HTTP control API; a provider with no backend at all may control it + in-process. + """ + + new_provider: ProviderFactory + """Creates the provider under test, against a backend that is already + running and seeded with the canonical flag set. Called once per scenario. + + Return a configured but uninitialised provider; the TCK initialises it. + """ + + new_unavailable_provider: ProviderFactory | None = None + """Creates a provider pointed at a backend that does not exist. + + Used by the initialisation-failure scenarios, which assert that a provider + unable to reach its backend settles into ``ERROR`` rather than hanging or + raising out of registration. + + Point it at a closed port on localhost. Do not point it at the backend under + test -- that must stay up, and simulated outages belong to :attr:`control`. + Configure a short connection deadline: the scenario allows a bounded time + for the error, and a provider with a 30-second connect timeout will not make + it. + + Required only if :attr:`capabilities` includes + :attr:`Capability.UNAVAILABLE_INIT`. Leaving both out is the honest + configuration for a provider with no backend, and those scenarios are then + skipped with the reason reported. + """ + + capabilities: Collection[Capability] = field(default=ALL_CAPABILITIES) + """Which optional parts of the provider contract this provider supports. + + Typed as a ``Collection`` rather than a ``frozenset`` so that the obvious + thing to write -- a set literal, which is what the README shows -- is also + the correctly typed thing to write. It is normalised to a frozenset on + construction, so a list, a set or a generator all behave identically. + + Scenarios tagged with an undeclared capability are reported as skipped with + the reason, never as passed. Defaults to everything; narrow it rather than + widening it. + """ + + event_timeout: float = DEFAULT_EVENT_TIMEOUT + """Seconds to wait for a provider event. + + The single most important knob for a provider author, because providers + observe backend changes on wildly different timescales. A streaming provider + sees a configuration change in milliseconds; one polling every 30 seconds + may need most of a poll interval. Set it to comfortably exceed your + worst-case detection latency, or the suite reports timeouts that are really + just impatience. + + Scenarios can tighten this with the explicit ``within {int}ms`` step, which + always wins over this value. + """ + + ready_timeout: float = DEFAULT_READY_TIMEOUT + """Seconds to wait for a provider to reach ``READY`` during initialisation.""" + + def __post_init__(self) -> None: + problems: list[str] = [] + + if not self.name: + problems.append( + "name is required: it scopes the OpenFeature domain and identifies " + "the suite in test output" + ) + if self.control is None: + problems.append( + "control is required: see BackendControl for which implementation " + "fits your provider" + ) + if self.new_provider is None: + problems.append( + "new_provider is required: the TCK has nothing to test without it" + ) + + # Normalise whatever iterable the caller passed into a frozenset, so a + # set literal, a list or a generator all behave the same. + object.__setattr__(self, "capabilities", frozenset(self.capabilities)) + + unknown = [c for c in self.capabilities if not isinstance(c, Capability)] + if unknown: + problems.append( + f"unknown capabilities {unknown!r}: capabilities are the members of " + f"the Capability enum" + ) + + if ( + Capability.UNAVAILABLE_INIT in self.capabilities + and self.new_unavailable_provider is None + ): + problems.append( + "capabilities declares Capability.UNAVAILABLE_INIT but " + "new_unavailable_provider is None: the @unavailable scenarios need a " + "provider pointed at a backend that does not exist. Supply one, or " + "remove the capability so those scenarios are skipped with a reason" + ) + + if problems: + joined = "\n - ".join(problems) + msg = f"invalid TckConfig:\n - {joined}" + raise ValueError(msg) + + @property + def domain(self) -> str: + """The OpenFeature domain this suite registers its providers under. + + Suite-scoped rather than scenario-scoped on purpose. Registering a new + provider in the same domain replaces the previous one; a fresh domain + per scenario would leave every provider of the suite registered, which + for a provider holding a network connection means leaking one connection + per scenario. + """ + return f"provider-tck/{self.name}" + + def declares(self, capability: Capability) -> bool: + return capability in self.capabilities + + @property + def sorted_capabilities(self) -> list[str]: + return sorted(c.tag for c in self.capabilities) + + +def capabilities_of(values: Iterable[Capability]) -> frozenset[Capability]: + """Convenience for building a capability set from any iterable.""" + return frozenset(values) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py new file mode 100644 index 00000000..0e83e5bd --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/control.py @@ -0,0 +1,114 @@ +"""The seam between the scenarios and whatever manipulates the backend.""" + +from __future__ import annotations + +import typing + +__all__ = [ + "BackendControl", + "ConnectionControl", + "UnsupportedControlError", + "unsupported_control", +] + + +class UnsupportedControlError(RuntimeError): + """Raised when a backend cannot perform a control operation. + + It is always a test-configuration bug rather than a provider defect. The + scenarios needing connection control are gated behind + :attr:`Capability.STALE` and :attr:`Capability.UNAVAILABLE_INIT`, so + reaching an unsupported operation means a capability was declared that the + backend cannot back up. The TCK fails loudly on it rather than skipping, + because a silent no-op would report the scenario as passed. + """ + + +@typing.runtime_checkable +class BackendControl(typing.Protocol): + """How the TCK puts the backend under test into the states a scenario needs. + + Step definitions never talk to a backend directly. They talk to this + protocol, which is why the same Gherkin runs unchanged against a + containerised backend driven over HTTP and against a provider manipulated + in-process. Nothing below this line knows about ports, containers or + transports. + + **Which implementation is right for your provider.** If your provider talks + to a backend -- a server, a service, anything out of process -- drive it + over the HTTP control API described in ``control-api.yaml``. That API is the + normative contract for those providers, and it is what makes a conformance + claim portable: another language's TCK drives the same endpoints against the + same stack and must get the same answers. + + Do not write an in-process control that reaches into an external backend + through a side channel -- a test-only admin client, a shared database + handle, a hook inside the provider. It will pass, and it will prove nothing, + because the path it exercised is not the path the contract describes. + + In-process control exists for providers with *no* backend to contract with: + in-memory, environment-variable and file-based providers, where "the + backend" is a data structure in the same process. See + :class:`InProcessControl`. + """ + + def prepare_scenario(self) -> None: + """Bring the backend to the state every scenario starts from. + + Reachable, with flag state at the baseline of the canonical flag set. + Called once before each scenario. This is the TCK's only isolation + mechanism -- scenarios share one backend for the whole suite, and + containers are never restarted between them. + """ + + def change_flag(self) -> None: + """Mutate flag configuration so a conforming provider observes a change. + + Afterwards the provider must resolve a different value for + ``changing-flag``. Which value it changes to is deliberately + unspecified; the suite asserts only that the resolved value differs. + """ + + @property + def description(self) -> str: + """A short description of what is being controlled, for messages a human reads.""" + + +@typing.runtime_checkable +class ConnectionControl(typing.Protocol): + """Implemented by a backend that can be cut off from the provider and restored. + + Separate from :class:`BackendControl` so a backend-less provider cannot + accidentally supply a no-op implementation: not implementing it at all is + the honest answer, and the TCK turns the resulting gap into an explicit, + reported skip. + """ + + def disconnect(self) -> None: + """Make the backend unreachable for the rest of the scenario, without stopping a container.""" + + def reconnect(self) -> None: + """Make the backend reachable again, preserving flag state. + + Preserving flag state is a requirement, not an implementation detail. An + outage must be observable as a change in availability and never as a + change in flag values, or the stale scenario cannot distinguish the two. + """ + + +def unsupported_control( + control: BackendControl, operation: str +) -> UnsupportedControlError: + """Build the error raised when a backend has no connection to control. + + The message names the fix, because the mistake it reports is always the same + one. + """ + return UnsupportedControlError( + f"{control.description} does not support {operation!r}. This is a " + f"test-configuration bug rather than a provider defect: a scenario needing " + f"connection control ran, so the suite declared Capability.STALE or " + f"Capability.UNAVAILABLE_INIT for a backend that cannot simulate an outage. " + f"Remove those capabilities from TckConfig.capabilities, or supply a " + f"BackendControl that also implements ConnectionControl." + ) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py new file mode 100644 index 00000000..1d69254c --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/inprocess.py @@ -0,0 +1,112 @@ +"""In-process backend control, for providers with no backend at all.""" + +from __future__ import annotations + +from openfeature.provider import FeatureProvider + +from .provider import ( + CHANGING_FLAG_KEY, + ControllableInMemoryProvider, + canonical_flag_set, + changing_flag, +) + +__all__ = ["InProcessControl"] + +_BASELINE = "foo" +_CHANGED = "bar" + + +class InProcessControl: + """Manipulates an in-process provider directly, with no backend and no HTTP. + + This exists so providers with nothing to connect to -- in-memory, + environment-variable and file-based providers -- can run the TCK. For those, + "the backend" is a data structure in the same process: seeding flags is + building a mapping, and changing one is an update on the live provider, so + the event the suite awaits is the provider's own + ``PROVIDER_CONFIGURATION_CHANGED`` rather than one the TCK synthesised. + + **This is not a shortcut for providers that do have a backend.** Reaching + into an external backend from inside the test process -- a test-only admin + client, a shared database handle, a hook in the provider -- produces a suite + that passes while proving nothing, because the path it exercised is not the + path the contract describes. Those providers drive the HTTP control API + instead. + + **Connection control.** :class:`InProcessControl` deliberately does not + implement :class:`~.control.ConnectionControl`. An in-memory provider has no + connection to lose, and pretending otherwise with a no-op would report the + ``@stale`` scenarios as passed. A suite using it leaves + :attr:`Capability.STALE` and :attr:`Capability.UNAVAILABLE_INIT` undeclared, + and those scenarios are skipped with the reason reported. + + **Ownership of the provider.** This type both seeds the flags and creates + the provider serving them, because in-process they are the same object: + :meth:`change_flag` has to reach the live instance to emit an event from it. + A suite therefore wires both through one control:: + + control = InProcessControl() + TckConfig( + name="in-memory", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.CONFIGURATION_CHANGE}, + ) + """ + + def __init__(self) -> None: + self._current: ControllableInMemoryProvider | None = None + self._changing_variant = _BASELINE + + @property + def description(self) -> str: + return "in-process control of an in-memory provider" + + def new_provider(self) -> FeatureProvider: + """Create the provider for the scenario about to run, at the baseline. + + Each call returns a fresh instance over a fresh copy of the canonical + flag set, which is what makes :meth:`prepare_scenario` nothing more than + dropping the previous reference. + """ + self._changing_variant = _BASELINE + self._current = ControllableInMemoryProvider(canonical_flag_set()) + return self._current + + def prepare_scenario(self) -> None: + """Drop the previous scenario's provider. + + That is the whole reset: the flag set is rebuilt per provider, so the + :meth:`new_provider` call that follows starts from an untouched + baseline. Clearing the reference rather than leaving it dangling means a + scenario that changes flags without creating a provider fails with a + clear message instead of mutating one that has already been shut down. + """ + self._current = None + + def change_flag(self) -> None: + """Flip ``changing-flag`` between its two variants on the live provider. + + The event the suite awaits is therefore the provider's own + ``PROVIDER_CONFIGURATION_CHANGED``, carrying ``changing-flag`` in + ``flags_changed``, and not a signal the TCK synthesised. + + Alternating rather than assigning a fixed variant keeps repeated calls + within one scenario meaningful; the suite asserts that the resolved + value differs, not what it became. + """ + if self._current is None: + msg = ( + "No in-memory provider exists for this scenario. In-process control " + "manipulates the provider itself, so the scenario must create one -- " + 'with "Given a stable provider" -- before any step that changes flag state.' + ) + raise RuntimeError(msg) + + self._changing_variant = ( + _BASELINE if self._changing_variant == _CHANGED else _CHANGED + ) + self._current.update_flag( + CHANGING_FLAG_KEY, changing_flag(self._changing_variant) + ) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py new file mode 100644 index 00000000..b8b1a73b --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/plugin.py @@ -0,0 +1,105 @@ +"""The pytest plugin: capability gating, scenario state, and the shared step vocabulary. + +Registered through the ``pytest11`` entry point, so installing this package is +all it takes for the step definitions to be available. pytest-bdd resolves steps +through the fixture system and fixtures from an installed plugin are visible to +every test, which is what keeps an adoption down to one fixture and one call to +:func:`tck_scenarios`. +""" + +from __future__ import annotations + +import typing + +import pytest + +from openfeature import api + +from .capability import Capability, capability_for_marker +from .config import TckConfig +from .state import TckState + +# The step modules are registered as plugins in their own right, not merely +# imported. pytest-bdd's decorators inject a generated fixture name into the +# *defining* module's namespace, so a step is only visible to pytest once the +# module defining it is a registered plugin -- importing it here would run the +# decorators but leave those fixtures where pytest never looks. +pytest_plugins = [ + "openfeature.contrib.tools.provider_tck.steps.provider_steps", + "openfeature.contrib.tools.provider_tck.steps.flag_steps", + "openfeature.contrib.tools.provider_tck.steps.event_steps", +] + + +def pytest_configure(config: pytest.Config) -> None: + """Register the capability tags as markers. + + pytest-bdd turns every Gherkin tag into a marker with + ``getattr(pytest.mark, tag)`` without registering it, which raises + ``PytestUnknownMarkWarning`` for each one -- noise at best, and a hard + failure in a project configured with ``-W error``. + """ + for capability in Capability: + config.addinivalue_line( + "markers", + f"{capability.value}: OpenFeature provider TCK capability {capability.tag}", + ) + + +@pytest.fixture +def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: + """Per-scenario state, carried between step definitions.""" + # Resetting here rather than in an autouse fixture ties the reset to the + # scenarios that actually use the TCK, and guarantees it happens after the + # capability gate has had its say -- a skipped scenario never touches the + # backend. + tck_config.control.prepare_scenario() + state = TckState(config=tck_config) + yield state + state.teardown() + + +@pytest.fixture(autouse=True) +def _tck_capability_gate(request: pytest.FixtureRequest) -> None: + """Skip a scenario whose capability the provider did not declare. + + ``pytest.skip`` here reports the scenario as skipped **with the reason**, + which is exactly what the specification asks a TCK implementation to do. + Nothing about it can be mistaken for a pass. + + The gate keys off the node's markers rather than its requested fixtures. + pytest-bdd resolves a step's fixtures lazily, as each step runs, so + ``tck_config`` is not in ``request.fixturenames`` when this autouse fixture + is set up -- guarding on that silently disabled the gate and let + ``@unavailable`` scenarios run against a config that never declared it. + + Checking markers first also means the gate costs nothing, and instantiates + nothing, for tests that are not TCK scenarios. + """ + gated = [ + capability + for marker in request.node.iter_markers() + if (capability := capability_for_marker(marker.name)) is not None + ] + if not gated: + return + + try: + config: TckConfig = request.getfixturevalue("tck_config") + except pytest.FixtureLookupError: + return + + for capability in gated: + if not config.declares(capability): + pytest.skip( + f"provider does not declare capability {capability.tag}. " + f"Declared: {' '.join(config.sorted_capabilities) or '(none)'}" + ) + + +@pytest.fixture(scope="session", autouse=True) +def _tck_release_providers() -> typing.Iterator[None]: + """Shut down whatever the suite registered once it is over.""" + yield + api.shutdown() + api.clear_providers() diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py new file mode 100644 index 00000000..33de6776 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/provider.py @@ -0,0 +1,134 @@ +"""An in-memory provider that can be reconfigured at runtime, and the canonical flag set.""" + +from __future__ import annotations + +import typing + +from openfeature.event import ProviderEventDetails +from openfeature.provider.in_memory_provider import ( + FlagStorage, + InMemoryFlag, + InMemoryProvider, +) + +__all__ = [ + "CHANGING_FLAG_KEY", + "ControllableInMemoryProvider", + "canonical_flag_set", + "changing_flag", +] + +CHANGING_FLAG_KEY = "changing-flag" +"""The flag :meth:`BackendControl.change_flag` mutates.""" + +_CHANGING_BASELINE = "foo" +_CHANGING_CHANGED = "bar" + + +class ControllableInMemoryProvider(InMemoryProvider): + """An in-memory provider whose flag set can be replaced at runtime. + + **Why this exists.** `Appendix A`_ of the specification requires an SDK's + in-memory provider to "support a means of updating the ``flag set``, + resulting in the emission of ``PROVIDER_CONFIGURATION_CHANGED`` events". The + Python SDK's :class:`~openfeature.provider.in_memory_provider.InMemoryProvider` + has no such method: it copies the flag mapping in its constructor and never + exposes a way to change it. + + Only half the machinery is missing, which is what makes this a small class + rather than a reimplementation. :class:`~openfeature.provider.AbstractProvider` + already supplies ``emit_provider_configuration_changed``, and the registry + already attaches the emitter, so all that is needed is a method that swaps + the mapping and emits. Everything about *resolution* -- variants, reasons, + ``FLAG_NOT_FOUND`` -- is still the SDK's. + + That makes this an honest reference for what the SDK's provider should grow, + rather than a competing implementation that could drift from it. + + .. _Appendix A: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md + """ + + def update_flags(self, flags: FlagStorage) -> None: + """Replace the whole flag set and emit a configuration-change event. + + The event names the union of the previous and new keys, which is what + Appendix A asks for: a consumer caching evaluations needs to know + everything that might have changed, and a key that disappeared has + changed as much as one that was added. + """ + changed = sorted(set(self._flags) | set(flags)) + self._flags = dict(flags) + self.emit_provider_configuration_changed( + ProviderEventDetails( + flags_changed=changed, message="flag configuration changed" + ) + ) + + def update_flag(self, key: str, flag: InMemoryFlag[typing.Any]) -> None: + """Replace a single flag and emit a configuration-change event naming it.""" + updated = dict(self._flags) + updated[key] = flag + self._flags = updated + self.emit_provider_configuration_changed( + ProviderEventDetails( + flags_changed=[key], message="flag configuration changed" + ) + ) + + def flag(self, key: str) -> InMemoryFlag[typing.Any] | None: + """Return the flag currently registered under ``key``.""" + return self._flags.get(key) + + +def changing_flag(default_variant: str) -> InMemoryFlag[str]: + return InMemoryFlag( + default_variant=default_variant, + variants={ + _CHANGING_BASELINE: _CHANGING_BASELINE, + _CHANGING_CHANGED: _CHANGING_CHANGED, + }, + ) + + +def canonical_flag_set() -> FlagStorage: + """Return the canonical flag set as SDK in-memory flags. + + Mirrors ``flag_data/canonical-flags.json`` entry for entry. Two properties + of that file are load-bearing and hold here too: + + * ``missing-flag`` is absent, which is what the ``FLAG_NOT_FOUND`` scenario + tests. Adding it turns that scenario green for the wrong reason. + * no flag carries a ``context_evaluator``, so every evaluation reports reason + ``STATIC`` -- the TCK tests a provider's mapping of a response, not a + backend's evaluation logic. + """ + return { + "boolean-flag": InMemoryFlag( + default_variant="on", variants={"on": True, "off": False} + ), + "string-flag": InMemoryFlag( + default_variant="greeting", variants={"greeting": "hi", "parting": "bye"} + ), + "integer-flag": InMemoryFlag( + default_variant="ten", variants={"one": 1, "ten": 10} + ), + "float-flag": InMemoryFlag( + default_variant="half", variants={"tenth": 0.1, "half": 0.5} + ), + "object-flag": InMemoryFlag( + default_variant="template", + variants={ + "empty": {}, + "template": { + "showImages": True, + "title": "Check out these pics!", + "imagesPerPage": 100, + }, + }, + ), + # A string flag, evaluated as a boolean by the TYPE_MISMATCH scenario. + "wrong-flag": InMemoryFlag( + default_variant="one", variants={"one": "uno", "two": "dos"} + ), + CHANGING_FLAG_KEY: changing_flag(_CHANGING_BASELINE), + } diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py new file mode 100644 index 00000000..71ea4150 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/state.py @@ -0,0 +1,146 @@ +"""Per-scenario state: what a scenario accumulates, and how it observes events. + +Separate from :mod:`plugin` so the step modules can import these types at the top +level. The step modules are loaded by the plugin as plugins in their own right, +and a step importing from the plugin module that loads it reads like a cycle even +where it is not one. +""" + +from __future__ import annotations + +import queue +import typing +from dataclasses import dataclass, field + +from openfeature.client import OpenFeatureClient +from openfeature.event import EventDetails, ProviderEvent +from openfeature.flag_evaluation import FlagType + +from .config import TckConfig + +__all__ = ["EvaluationRecord", "EventRecorder", "TckState"] + + +@dataclass +class EvaluationRecord: + """The outcome of one flag evaluation, flattened across the five typed calls.""" + + value: typing.Any = None + variant: str | None = None + reason: str | None = None + error_code: str | None = None + error_message: str | None = None + raised: BaseException | None = None + """The exception the call raised, if any. + + In Python an errored evaluation returns the code default in the details + rather than raising, so this stays ``None`` on the error paths the suite + exercises. It is what "no exception should have been thrown" asserts. + """ + + +class EventRecorder: + """Captures the events of one type, in order, so a scenario consumes them one at a time. + + Consuming rather than merely observing is what makes the stale scenario + work: it awaits a ``PROVIDER_READY`` at the start and a second, different + ``PROVIDER_READY`` once the backend is back, and a recorder that only + remembered "ready has fired at some point" would report the second assertion + as satisfied by the first event. + + A queue rather than a list because a provider with a background thread -- + anything with a real backend -- delivers events from that thread while the + scenario waits on the main one. + """ + + def __init__(self, client: OpenFeatureClient, event: ProviderEvent) -> None: + self.event = event + self._client = client + self._events: queue.Queue[EventDetails] = queue.Queue() + self.last: EventDetails | None = None + + # The SDK replays a matching event on registration when the provider is + # already in the corresponding state, so a handler added after the + # provider became ready still observes its PROVIDER_READY. That is what + # lets the feature files register handlers after "Given a stable + # provider" without racing it. + client.add_handler(event, self._on_event) + + def _on_event(self, details: EventDetails) -> None: + self._events.put(details) + + def await_event(self, timeout: float) -> EventDetails: + """Consume the next event of this recorder's type.""" + try: + details = self._events.get(timeout=timeout) + except queue.Empty: + msg = ( + f"timed out after {timeout}s waiting for a {self.event.value} event. " + f"If the provider is simply slower than this to notice, raise " + f"TckConfig.event_timeout rather than treating it as a failure" + ) + raise AssertionError(msg) from None + self.last = details + return details + + def detach(self) -> None: + self._client.remove_handler(self.event, self._on_event) + + +@dataclass +class TckState: + """Everything one scenario accumulates.""" + + config: TckConfig + client: OpenFeatureClient | None = None + flag_key: str | None = None + flag_type: FlagType | None = None + default_value: typing.Any = None + last: EvaluationRecord | None = None + remembered: typing.Any = None + has_memory: bool = False + recorders: dict[ProviderEvent, EventRecorder] = field(default_factory=dict) + + def require_client(self) -> OpenFeatureClient: + if self.client is None: + msg = ( + "no provider has been registered in this scenario: a " + '"Given a stable provider" or "Given a unavailable provider" step ' + "must come first" + ) + raise AssertionError(msg) + return self.client + + def require_flag(self) -> tuple[str, FlagType, typing.Any]: + if self.flag_key is None or self.flag_type is None: + msg = ( + "no flag has been declared in this scenario: a " + '"Given a -flag with key ... and a default value ..." step ' + "must come first" + ) + raise AssertionError(msg) + return self.flag_key, self.flag_type, self.default_value + + def require_evaluation(self) -> EvaluationRecord: + if self.last is None: + msg = ( + "no flag has been evaluated in this scenario: a " + '"When the flag was evaluated with details" step must come first' + ) + raise AssertionError(msg) + return self.last + + def require_recorder(self, event: ProviderEvent) -> EventRecorder: + recorder = self.recorders.get(event) + if recorder is None: + msg = ( + f"no handler was registered for {event.value} in this scenario: a " + '"Given a event handler" step must come first' + ) + raise AssertionError(msg) + return recorder + + def teardown(self) -> None: + for recorder in self.recorders.values(): + recorder.detach() + self.recorders.clear() diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/__init__.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/__init__.py new file mode 100644 index 00000000..6581ee86 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/__init__.py @@ -0,0 +1,11 @@ +"""The shared step vocabulary. + +Each module here is registered as a pytest plugin by the TCK's own plugin, which +is what makes the steps visible: pytest-bdd's decorators inject a generated +fixture name into the *defining* module's namespace, so a step only reaches +pytest once its module is a registered plugin. + +Deliberately empty of imports. Pulling the submodules in here would import them +before pytest loads them as plugins, and pytest cannot rewrite assertions in a +module that is already imported. +""" diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py new file mode 100644 index 00000000..6b635699 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/event_steps.py @@ -0,0 +1,165 @@ +"""Steps covering provider events, connection loss and client status.""" + +from __future__ import annotations + +from pytest_bdd import given, parsers, then, when + +from openfeature.event import ProviderEvent +from openfeature.provider import ProviderStatus + +from ..control import ConnectionControl, unsupported_control +from ..state import EventRecorder, TckState + +__all__ = [ + "an_event_handler", + "an_event_was_fired", + "the_client_should_be_in_state", + "the_connection_is_lost", + "the_connection_is_restored", + "the_event_handler_should_have_been_executed", + "the_event_handler_should_have_been_executed_within", + "the_flag_should_be_part_of_the_event_payload", +] + +_EVENT_BY_NAME: dict[str, ProviderEvent] = { + "ready": ProviderEvent.PROVIDER_READY, + "stale": ProviderEvent.PROVIDER_STALE, + "error": ProviderEvent.PROVIDER_ERROR, + "change": ProviderEvent.PROVIDER_CONFIGURATION_CHANGED, +} + +_STATUS_BY_NAME: dict[str, ProviderStatus] = { + "ready": ProviderStatus.READY, + "stale": ProviderStatus.STALE, + "error": ProviderStatus.ERROR, +} + + +def _event(name: str) -> ProviderEvent: + try: + return _EVENT_BY_NAME[name] + except KeyError: + msg = f"unknown event kind {name!r}" + raise AssertionError(msg) from None + + +@given(parsers.re(r"^an? (?Pready|stale|error|change) event handler$")) +def an_event_handler(tck_state: TckState, kind: str) -> None: + """Attach a recorder for one event type. + + Handlers are attached after the provider is registered, which the SDK + handles by replaying a matching event on registration when the provider is + already in the corresponding state. That is why "Given a stable provider" + followed by "And a ready event handler" is not a race. + """ + event = _event(kind) + if event in tck_state.recorders: + return + client = tck_state.require_client() + tck_state.recorders[event] = EventRecorder(client, event) + + +@when(parsers.re(r"^a (?Pready|stale|error|change) event was fired$")) +def an_event_was_fired(tck_state: TckState, kind: str) -> None: + """Consume an event, so a later assertion observes the next one rather than this. + + The stale scenario depends on it: it consumes the initial ``PROVIDER_READY`` + here and then asserts a second, distinct one once the backend is back. + """ + recorder = tck_state.require_recorder(_event(kind)) + recorder.await_event(tck_state.config.event_timeout) + + +@then( + parsers.re( + r"^the (?Pready|stale|error|change) event handler should have been executed$" + ) +) +def the_event_handler_should_have_been_executed(tck_state: TckState, kind: str) -> None: + recorder = tck_state.require_recorder(_event(kind)) + recorder.await_event(tck_state.config.event_timeout) + + +@then( + parsers.re( + r"^the (?Pready|stale|error|change) event handler should have been " + r"executed within (?P\d+)ms$" + ) +) +def the_event_handler_should_have_been_executed_within( + tck_state: TckState, kind: str, millis: str +) -> None: + """Bound the wait explicitly. + + The scenarios using this assert promptness, not merely eventual arrival: a + provider that cannot reach its backend has to report that fact quickly, + because an application blocked on provider registration is down. The bound + therefore overrides ``event_timeout`` rather than being clamped by it. + """ + recorder = tck_state.require_recorder(_event(kind)) + recorder.await_event(int(millis) / 1000.0) + + +@then("the flag should be part of the event payload") +def the_flag_should_be_part_of_the_event_payload(tck_state: TckState) -> None: + """Assert the configuration-change event named the flag that changed. + + Naming the changed flags is what makes the event actionable: a consumer + caching evaluations needs to know what to invalidate, and an event carrying + no keys forces it to invalidate everything. + """ + key, _flag_type, _default = tck_state.require_flag() + recorder = tck_state.require_recorder(ProviderEvent.PROVIDER_CONFIGURATION_CHANGED) + + if recorder.last is None: + msg = ( + "no configuration-change event has been consumed in this scenario: a " + '"the change event handler should have been executed" step must come first' + ) + raise AssertionError(msg) + + changed = recorder.last.flags_changed or [] + if key in changed: + return + + if not changed: + msg = ( + f"the configuration-change event carried no changed flags, expected it to " + f"name {key!r}" + ) + else: + msg = f"the configuration-change event named {changed}, expected it to include {key!r}" + raise AssertionError(msg) + + +def _connection_control(tck_state: TckState, operation: str) -> ConnectionControl: + control = tck_state.config.control + if not isinstance(control, ConnectionControl): + raise unsupported_control(control, operation) + return control + + +@when("the connection is lost") +def the_connection_is_lost(tck_state: TckState) -> None: + _connection_control(tck_state, "disconnect").disconnect() + + +@when("the connection is restored") +def the_connection_is_restored(tck_state: TckState) -> None: + _connection_control(tck_state, "reconnect").reconnect() + + +@then(parsers.re(r"^the client should be in (?Pready|stale|error) state$")) +def the_client_should_be_in_state(tck_state: TckState, name: str) -> None: + """Assert the provider status the client reports. + + Checked after the corresponding event has been consumed, and the SDK writes + provider status before running handlers, so no polling is needed: if the + event arrived, the status is already current. + """ + client = tck_state.require_client() + expected = _STATUS_BY_NAME[name] + actual = client.get_provider_status() + if actual != expected: + msg = f"client reports status {actual}, expected {expected}" + raise AssertionError(msg) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py new file mode 100644 index 00000000..f284eb5b --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/flag_steps.py @@ -0,0 +1,240 @@ +"""Steps that declare, evaluate and assert flags.""" + +from __future__ import annotations + +import typing +from collections.abc import Callable + +from pytest_bdd import given, parsers, then, when + +from openfeature.flag_evaluation import FlagType + +from ..state import EvaluationRecord, TckState +from ..values import describe, parse_flag_type, parse_value, values_equal + +__all__ = [ + "a_flag_with_key_and_default", + "no_exception_should_have_been_thrown", + "the_error_code_should_be", + "the_flag_was_evaluated_with_details", + "the_flag_was_modified", + "the_reason_should_be", + "the_resolved_object_value_should_contain", + "the_resolved_value_is_remembered", + "the_resolved_value_should_be", + "the_resolved_value_should_have_changed", + "the_variant_should_be", +] + + +@given( + parsers.re( + r'^an? (?P[A-Za-z]+)-flag with key "(?P[^"]*)" ' + r'and a default value "(?P[^"]*)"$' + ) +) +def a_flag_with_key_and_default( + tck_state: TckState, flag_type: str, key: str, default: str +) -> None: + """Declare the flag the scenario is about, and the type it is requested as. + + The two are independent on purpose: most of ``errors.feature`` asks for a + flag as a type it is not. + """ + parsed_type = parse_flag_type(flag_type) + tck_state.flag_key = key + tck_state.flag_type = parsed_type + tck_state.default_value = parse_value(parsed_type, default) + + +@when("the flag was evaluated with details") +def the_flag_was_evaluated_with_details(tck_state: TckState) -> None: + """Resolve the declared flag through the typed client call matching its type.""" + client = tck_state.require_client() + key, flag_type, default = tck_state.require_flag() + + # Annotated explicitly: the five typed getters have different signatures, so + # an unannotated mapping infers a value type mypy will not let us call. + calls: dict[FlagType, Callable[[str, typing.Any], typing.Any]] = { + FlagType.BOOLEAN: client.get_boolean_details, + FlagType.STRING: client.get_string_details, + FlagType.INTEGER: client.get_integer_details, + FlagType.FLOAT: client.get_float_details, + FlagType.OBJECT: client.get_object_details, + } + + record = EvaluationRecord() + try: + details = calls[flag_type](key, default) + except BaseException as exc: # recorded here, asserted on by its own step + record.raised = exc + record.value = default + else: + record.value = details.value + record.variant = details.variant + record.reason = str(details.reason) if details.reason is not None else None + record.error_code = ( + details.error_code.value if details.error_code is not None else None + ) + record.error_message = details.error_message + + tck_state.last = record + + +@then(parsers.re(r'^the resolved details value should be "(?P[^"]*)"$')) +def the_resolved_value_should_be(tck_state: TckState, expected: str) -> None: + _key, flag_type, _default = tck_state.require_flag() + record = tck_state.require_evaluation() + wanted = parse_value(flag_type, expected) + + if not values_equal(wanted, record.value): + detail = ( + f" (the client also reported: {record.error_message})" + if record.error_message + else "" + ) + msg = ( + f"flag {tck_state.flag_key!r} resolved to {describe(record.value)}, " + f"expected {describe(wanted)}{detail}" + ) + raise AssertionError(msg) + + +@then(parsers.re(r'^the variant should be "(?P[^"]*)"$')) +def the_variant_should_be(tck_state: TckState, expected: str) -> None: + record = tck_state.require_evaluation() + if record.variant != expected: + msg = ( + f"variant was {record.variant!r}, expected {expected!r}. A variant that " + f"does not survive the trip from the backend is one of the easiest parts " + f"of the contract to drop" + ) + raise AssertionError(msg) + + +@then(parsers.re(r'^the reason should be "(?P[^"]*)"$')) +def the_reason_should_be(tck_state: TckState, expected: str) -> None: + record = tck_state.require_evaluation() + if record.reason != expected: + msg = f"reason was {record.reason!r}, expected {expected!r}" + raise AssertionError(msg) + + +@then(parsers.re(r'^the error-code should be "(?P[^"]*)"$')) +def the_error_code_should_be(tck_state: TckState, expected: str) -> None: + """Assert the reported error code, where the empty string means none at all. + + The empty case matters as much as the populated ones. A provider that + reports a plausible value with no error code is the failure mode the suite + is most concerned with, because the application has no way to notice. + """ + record = tck_state.require_evaluation() + actual = record.error_code or "" + + if actual == expected: + return + + if expected == "": + msg = f"error-code was {actual!r}, expected none" + elif actual == "": + msg = ( + f"no error-code was reported, expected {expected!r}. Returning a value " + f"without an error code leaves the application unable to tell that " + f"anything went wrong" + ) + else: + msg = f"error-code was {actual!r}, expected {expected!r}" + raise AssertionError(msg) + + +@then("no exception should have been thrown") +def no_exception_should_have_been_thrown(tck_state: TckState) -> None: + """Assert the evaluation returned rather than raised. + + In Python an errored evaluation returns the code default in the details and + does not raise, so this holds on the error paths too. A provider that raises + instead takes the calling application down with it, which is what the + feature files forbid. + """ + record = tck_state.require_evaluation() + if record.raised is not None: + msg = ( + f"the evaluation raised {record.raised!r}. A flag evaluation must always " + f"return a value and an error code, never raise" + ) + raise AssertionError(msg) + + +@then("the resolved object value should contain") +def the_resolved_object_value_should_contain( + tck_state: TckState, datatable: list[list[str]] +) -> None: + """Assert members of a structured value, each with its own expected type.""" + record = tck_state.require_evaluation() + header, *rows = datatable + + if header != ["key", "type", "value"]: + msg = f"expected a data table with columns key, type, value; got {header}" + raise AssertionError(msg) + + if not isinstance(record.value, dict): + msg = ( + f"resolved object value is {describe(record.value)}, which has no members " + f"to check" + ) + raise AssertionError(msg) + + for key, raw_type, raw_value in rows: + wanted = parse_value(parse_flag_type(raw_type), raw_value) + if key not in record.value: + msg = f"resolved object value has no member {key!r}" + raise AssertionError(msg) + actual = record.value[key] + if not values_equal(wanted, actual): + msg = f"object member {key!r} was {describe(actual)}, expected {describe(wanted)}" + raise AssertionError(msg) + + +@when("the resolved value is remembered") +def the_resolved_value_is_remembered(tck_state: TckState) -> None: + """Store the current value so a later step can assert it changed.""" + record = tck_state.require_evaluation() + tck_state.remembered = record.value + tck_state.has_memory = True + + +@then("the resolved details value should have changed") +def the_resolved_value_should_have_changed(tck_state: TckState) -> None: + """Assert that re-evaluation produced a different value. + + This is the half of the configuration-change contract providers actually get + wrong. Emitting ``PROVIDER_CONFIGURATION_CHANGED`` and then continuing to + resolve the old value is worse than emitting nothing, because the + application acted on a signal that was not true. + """ + record = tck_state.require_evaluation() + if not tck_state.has_memory: + msg = ( + "no value was remembered in this scenario: a " + '"the resolved value is remembered" step must come first' + ) + raise AssertionError(msg) + + if values_equal(tck_state.remembered, record.value): + msg = ( + f"the resolved value is still {describe(record.value)} after the " + f"configuration changed. The change was signalled but not applied, so the " + f"event told the application something untrue" + ) + raise AssertionError(msg) + + +@when("the flag was modified") +def the_flag_was_modified(tck_state: TckState) -> None: + """Change flag configuration on the backend.""" + control = tck_state.config.control + try: + control.change_flag() + except Exception as exc: + msg = f"could not change flag configuration on {control.description}: {exc}" + raise AssertionError(msg) from exc diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py new file mode 100644 index 00000000..bca37fae --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/steps/provider_steps.py @@ -0,0 +1,114 @@ +"""Steps that put a provider under test.""" + +from __future__ import annotations + +import concurrent.futures +import contextlib + +from pytest_bdd import given, parsers + +from openfeature import api +from openfeature.provider import FeatureProvider + +from ..state import TckState + +__all__ = ["a_stable_provider", "an_unavailable_provider"] + + +@given(parsers.re(r"^an? stable provider$")) +def a_stable_provider(tck_state: TckState) -> None: + """Register the provider under test against the running, seeded backend. + + ``api.set_provider`` initialises synchronously and dispatches + ``PROVIDER_READY``, so by the time this step returns the provider is ready + and every scenario that follows can assume it. A suite that started + evaluating before that would report races in the TCK as defects in the + provider. + """ + config = tck_state.config + provider = config.new_provider() + if provider is None: + msg = "TckConfig.new_provider returned None" + raise AssertionError(msg) + + try: + _set_provider_within(provider, config.domain, config.ready_timeout) + except TimeoutError: + msg = ( + f"the provider did not become ready within {config.ready_timeout}s. The backend " + f"is up and seeded at this point, so either initialisation is genuinely hanging " + f"or TckConfig.ready_timeout is too short" + ) + raise AssertionError(msg) from None + except Exception as exc: + msg = ( + f"registering the provider raised {exc!r}. The backend is up and seeded " + f"at this point, so this is a genuine initialisation failure rather than " + f"the unavailable-backend case" + ) + raise AssertionError(msg) from exc + + tck_state.client = api.get_client(config.domain) + + +@given(parsers.re(r"^an? unavailable provider$")) +def an_unavailable_provider(tck_state: TckState) -> None: + """Register a provider pointed at a backend that does not exist. + + Neither a failed initialisation nor a raised exception during registration + is a failure here: what the contract requires is that the provider settles + into an observable error state promptly, which the scenario asserts through + the event and the client status. The SDK's registry already converts a + raising ``initialize`` into ``PROVIDER_ERROR``, so registration itself is + expected to return normally -- but a provider that raises anyway must not + take the scenario down with it, which is why this is caught rather than + propagated. + """ + config = tck_state.config + + if config.new_unavailable_provider is None: + msg = ( + "TckConfig.new_unavailable_provider is None but an @unavailable scenario " + "ran. This is a test-configuration bug rather than a provider defect: the " + "suite declared Capability.UNAVAILABLE_INIT without supplying a provider " + "that cannot reach its backend. Remove that capability, or supply the factory" + ) + raise AssertionError(msg) + + provider = config.new_unavailable_provider() + if provider is None: + msg = "TckConfig.new_unavailable_provider returned None" + raise AssertionError(msg) + + # A raising initialize is already converted to PROVIDER_ERROR by the SDK's + # registry, so this is belt and braces: a provider that raises anyway must + # not take the scenario down with it, because the contract is about the + # observable error state rather than about how registration returned. + with contextlib.suppress(Exception): + api.set_provider(provider, config.domain) + + tck_state.client = api.get_client(config.domain) + + +def _set_provider_within( + provider: FeatureProvider, domain: str, timeout: float +) -> None: + """Register a provider, giving up if initialisation has not returned in time. + + ``api.set_provider`` initialises synchronously and has no timeout of its own, so a + provider that hangs while connecting would hang the whole session with no useful + message. Running it on a worker thread bounds it. + + The worker is deliberately not cancelled on timeout -- Python cannot interrupt a + thread blocked in a socket call -- so it is left to finish or die with the process. + That is acceptable here because a timeout already means the scenario is failing. + """ + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(api.set_provider, provider, domain) + try: + future.result(timeout=timeout) + except concurrent.futures.TimeoutError: + raise TimeoutError from None + finally: + # Do not block __exit__ on a worker that is still stuck. + pool.shutdown(wait=False) diff --git a/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py new file mode 100644 index 00000000..4459d466 --- /dev/null +++ b/tools/openfeature-provider-tck/src/openfeature/contrib/tools/provider_tck/values.py @@ -0,0 +1,125 @@ +"""Turning Gherkin strings into typed values, and comparing them with what a provider resolved.""" + +from __future__ import annotations + +import json +import typing + +from openfeature.flag_evaluation import FlagType + +__all__ = ["describe", "parse_flag_type", "parse_value", "values_equal"] + +_BY_NAME: dict[str, FlagType] = { + "boolean": FlagType.BOOLEAN, + "string": FlagType.STRING, + "integer": FlagType.INTEGER, + "float": FlagType.FLOAT, + "object": FlagType.OBJECT, +} + + +def parse_flag_type(raw: str) -> FlagType: + """Resolve the type named in a scenario, case-insensitively.""" + try: + return _BY_NAME[raw.strip().lower()] + except KeyError: + names = ", ".join(sorted(n.capitalize() for n in _BY_NAME)) + msg = f"unknown flag type {raw!r}: expected one of {names}" + raise ValueError(msg) from None + + +def _parse_bool(raw: str) -> bool: + lowered = raw.strip().lower() + if lowered in {"true", "t", "yes", "1"}: + return True + if lowered in {"false", "f", "no", "0"}: + return False + msg = f"{raw!r} is not a boolean" + raise ValueError(msg) + + +def parse_value(flag_type: FlagType, raw: str) -> typing.Any: + """Convert a value written in a scenario into the type the API uses. + + Everything in Gherkin is a string, so this is where ``"0.5"`` becomes a + float and ``"{}"`` becomes an empty object. Parsing per declared type rather + than guessing is what keeps the integer and float scenarios + distinguishable: ``"1"`` is an ``int`` in an Integer scenario and a ``float`` + in a Float one. + """ + if flag_type is FlagType.BOOLEAN: + return _parse_bool(raw) + if flag_type is FlagType.STRING: + return raw + if flag_type is FlagType.INTEGER: + return int(raw) + if flag_type is FlagType.FLOAT: + return float(raw) + if flag_type is FlagType.OBJECT: + # Gherkin escapes quotes in table cells; pytest-bdd keeps the backslash, + # so strip it before handing the text to json. + return json.loads(raw.replace('\\"', '"')) + msg = f"unknown flag type {flag_type!r}" + raise ValueError(msg) + + +def _as_number(value: typing.Any) -> float | None: + """Return a numeric value as a float, or None if it is not numeric. + + Booleans are deliberately excluded. Python makes ``bool`` a subclass of + ``int``, so an unguarded numeric comparison would quietly report ``True`` and + ``1`` as equal -- which is the exact confusion several of these scenarios + exist to detect. + """ + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + return None + + +def values_equal(expected: typing.Any, actual: typing.Any) -> bool: + """Compare an expected value from a scenario with what a provider resolved. + + Numbers are compared numerically rather than by Python type. A provider that + deserialises its backend's JSON hands back ``float`` for every number, so the + ``100`` inside ``object-flag`` arrives as ``100.0`` from one provider and + ``100`` from another while both are correct. Type distinctness is asserted + where it belongs -- by requesting a flag as a specific type and checking the + error code -- not by accident of how a number was decoded. + """ + # A boolean only ever equals a boolean. Without this, Python's bool-is-an-int + # rule would make True == 1 and quietly satisfy the scenario that exists to + # catch exactly that confusion. + if isinstance(expected, bool) or isinstance(actual, bool): + return ( + isinstance(expected, bool) + and isinstance(actual, bool) + and expected == actual + ) + + expected_number = _as_number(expected) + if expected_number is not None: + actual_number = _as_number(actual) + return actual_number is not None and expected_number == actual_number + + if isinstance(expected, dict) and isinstance(actual, dict): + if set(expected) != set(actual): + return False + return all(values_equal(v, actual[k]) for k, v in expected.items()) + + if isinstance(expected, list) and isinstance(actual, list): + return len(expected) == len(actual) and all( + values_equal(e, a) for e, a in zip(expected, actual, strict=True) + ) + + return bool(expected == actual) + + +def describe(value: typing.Any) -> str: + """Render a value for a failure message, including its type. + + "expected 100 but got 100" is the single most confusing failure a + cross-language conformance suite can produce. + """ + return f"{value!r} ({type(value).__name__})" diff --git a/tools/openfeature-provider-tck/tests/conftest.py b/tools/openfeature-provider-tck/tests/conftest.py new file mode 100644 index 00000000..a5e6726f --- /dev/null +++ b/tools/openfeature-provider-tck/tests/conftest.py @@ -0,0 +1,39 @@ +"""Known deviations of the Python SDK, recorded rather than hidden. + +A conformance suite that quietly goes green on scenarios it did not run is worse +than no suite at all -- and the same is true of one that quietly goes green on a +scenario it *did* run and fail. So the one scenario the Python SDK cannot +currently satisfy is marked ``xfail(strict=True)`` here, which: + +* keeps it visible in the report, as XFAIL with the reason attached; +* fails the suite if it ever *passes*, so the marker is removed the moment the + SDK is fixed rather than lingering as a lie. + +This lives in the TCK's own self-test rather than in the shared package. It is a +fact about the SDK under test, not part of the conformance definition, and +Appendix F deliberately leaves a general "known deviations" concept as an open +question (spec#417, Q4). If that concept lands, this moves into it. +""" + +from __future__ import annotations + +import pytest + +# The Scenario Outline row that asks for boolean-flag as an Integer. +_BOOL_AS_INT = ( + "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" +) + +_REASON = ( + "python-sdk: a boolean satisfies an Integer request. The client type-checks with " + "isinstance(value, int) and bool is a subclass of int in Python, so boolean-flag " + "requested as an Integer returns True with reason STATIC and no error code, where " + "the specification requires the code default and TYPE_MISMATCH. " + "See https://github.com/open-feature/python-sdk/issues/619" +) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + for item in items: + if item.name == _BOOL_AS_INT: + item.add_marker(pytest.mark.xfail(reason=_REASON, strict=True)) diff --git a/tools/openfeature-provider-tck/tests/test_controllable_conformance.py b/tools/openfeature-provider-tck/tests/test_controllable_conformance.py new file mode 100644 index 00000000..f127243c --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_controllable_conformance.py @@ -0,0 +1,56 @@ +"""Run the conformance suite against the TCK's own updatable in-memory provider. + +This is the suite that exercises the configuration-change path, and it exists +because the SDK's in-memory provider cannot: it has no way to update a flag set, +so ``test_in_memory_conformance`` necessarily skips those scenarios. Without +this suite the change-event step definitions would ship with no coverage at all, +and a break in them would first surface in a containerised provider suite where +it looks like a provider defect. + +It is also the reference for what an in-process control path looks like when the +provider does support updates, which is what a file-based or +environment-variable provider should be able to do. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + InProcessControl, + TckConfig, + features_path, +) + + +@pytest.fixture(scope="session") +def tck_config() -> TckConfig: + """Declare the provider under test and what it can do. + + ``STALE`` and ``UNAVAILABLE_INIT`` stay undeclared: there is still no + connection to lose, and ``InProcessControl`` does not implement + ``ConnectionControl``. ``CONFIGURATION_CHANGE`` is what this suite adds over + the plain in-memory one, and it is the whole point of it. + + ``LIFECYCLE`` stays undeclared for the same reason as in + ``test_in_memory_conformance``: there is no backend to reach during + initialisation, so the readiness scenario would pass here without testing + anything. It did exactly that while the feature was gated on ``@events``. + """ + control = InProcessControl() + return TckConfig( + name="controllable-in-memory", + control=control, + new_provider=control.new_provider, + capabilities={ + Capability.EVENTS, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.STRICT_NUMERIC_TYPING, + }, + ) + + +scenarios(features_path()) diff --git a/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py new file mode 100644 index 00000000..648ba05a --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_in_memory_conformance.py @@ -0,0 +1,109 @@ +"""Run the conformance suite against the SDK's own in-memory provider. + +This is the TCK's self-test, and it earns its keep twice over. + +It is the **reference adoption** for a provider with no backend. Everything a +file-based or environment-variable provider has to write is here: one fixture +and one call. + +It is also the **Docker-free canary**. Needing no container and no network, it +runs in a fraction of a second, which makes it the fast check that catches a +broken step definition, a mis-wired capability gate or a regression in the +shared harness long before a containerised suite would. + +What it does not do is license providers that have a backend to test themselves +this way -- see ``BackendControl`` for why. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.provider_tck import ( + Capability, + TckConfig, + canonical_flag_set, + features_path, +) +from openfeature.provider import FeatureProvider +from openfeature.provider.in_memory_provider import InMemoryProvider + + +class PlainMemoryControl: + """Backend control for the SDK's stock in-memory provider. + + ``prepare_scenario`` is a no-op because the provider is rebuilt from the + canonical flag set for every scenario, so each one already starts from an + untouched baseline. + + ``change_flag`` cannot be implemented at all, and the error says why. + Appendix A of the specification requires an SDK's in-memory provider to + "support a means of updating the flag set, resulting in the emission of + PROVIDER_CONFIGURATION_CHANGED events"; the Python SDK's copies its mapping + in the constructor and exposes no way to change it. The suite below + therefore leaves ``CONFIGURATION_CHANGE`` undeclared and the scenario is + reported as skipped with its reason, which is the honest outcome. Reaching + this error would mean the capability had been declared anyway. + """ + + @property + def description(self) -> str: + return "the Python SDK's InMemoryProvider, rebuilt per scenario" + + def prepare_scenario(self) -> None: + return None + + def change_flag(self) -> None: + msg = ( + "openfeature.provider.in_memory_provider.InMemoryProvider cannot change its " + "flag set: it copies the mapping in its constructor and exposes no update " + "method, so a configuration change can be neither applied nor signalled. " + "Appendix A of the specification requires it. See " + "ControllableInMemoryProvider for what the SDK's provider is missing" + ) + raise NotImplementedError(msg) + + +def _new_provider() -> FeatureProvider: + return InMemoryProvider(canonical_flag_set()) + + +@pytest.fixture(scope="session") +def tck_config() -> TckConfig: + """Declare the provider under test and what it can do. + + Each omission is a fact about the provider rather than a convenience: + + * ``CONFIGURATION_CHANGE`` -- omitted because the SDK's in-memory provider + cannot update its flag set. That is a finding, not a configuration choice; + see ``PlainMemoryControl``. + * ``STALE`` and ``UNAVAILABLE_INIT`` -- omitted because there is no + connection to lose. ``PlainMemoryControl`` does not implement + ``ConnectionControl`` for the same reason, and the two omissions keep each + other honest: the scenarios are skipped before any step can reach an + operation the control cannot perform. + * ``TARGETING`` and ``CACHING`` -- omitted because no scenario carries their + tags yet, so leaving them out skips nothing. + * ``LIFECYCLE`` -- omitted because there is no backend to reach. The + capability asserts that initialisation actually contacts a backend and + that the outcome is observable; this provider's ``initialize`` is a no-op + and the SDK dispatches ``PROVIDER_READY`` around it regardless, so the + readiness scenario would pass here without testing anything. It passed + vacuously while the feature was gated on ``EVENTS``, which is precisely + the failure mode the split of ``@lifecycle`` from ``@events`` exists to + end. A skip with a reason is the honest outcome. + """ + return TckConfig( + name="in-memory", + control=PlainMemoryControl(), + new_provider=_new_provider, + capabilities={ + Capability.EVENTS, + Capability.OBJECT, + Capability.STRICT_NUMERIC_TYPING, + }, + ) + + +scenarios(features_path()) diff --git a/tools/openfeature-provider-tck/tests/test_in_process_control.py b/tools/openfeature-provider-tck/tests/test_in_process_control.py new file mode 100644 index 00000000..88322acf --- /dev/null +++ b/tools/openfeature-provider-tck/tests/test_in_process_control.py @@ -0,0 +1,141 @@ +"""Things the Gherkin cannot assert about itself. + +Each of these is a way the in-process control path could look correct while +quietly making the conformance suites meaningless. +""" + +from __future__ import annotations + +import pytest + +from openfeature.contrib.tools.provider_tck import ( + CHANGING_FLAG_KEY, + ConnectionControl, + ControllableInMemoryProvider, + InProcessControl, + canonical_flag_set, +) +from openfeature.event import ProviderEvent + + +def _resolve_changing(provider: ControllableInMemoryProvider) -> str: + return provider.resolve_string_details(CHANGING_FLAG_KEY, "unset").value + + +def test_change_flag_actually_changes_the_resolved_value() -> None: + """The assumption every configuration-change scenario rests on. + + If ``change_flag`` emitted an event without altering what the provider + resolves, the scenario would still pass its event assertion and the suite + would be certifying a signal with nothing behind it. + """ + control = InProcessControl() + provider = control.new_provider() + assert isinstance(provider, ControllableInMemoryProvider) + + before = _resolve_changing(provider) + control.change_flag() + after = _resolve_changing(provider) + + assert before != after, "change_flag did not change the resolved value" + + +def test_change_flag_emits_a_configuration_change_event_naming_the_flag() -> None: + """The event the scenarios await is the provider's own, and it names the flag.""" + control = InProcessControl() + provider = control.new_provider() + + seen: list[tuple[ProviderEvent, list[str] | None]] = [] + + def record(_provider: object, event: ProviderEvent, details: object) -> None: + seen.append((event, getattr(details, "flags_changed", None))) + + # attach() is how the SDK registry wires a provider's emitter; doing it by + # hand keeps this a unit test of the provider rather than of the registry. + provider.attach(record) + control.change_flag() + + assert seen, "no event was emitted" + event, flags_changed = seen[-1] + assert event is ProviderEvent.PROVIDER_CONFIGURATION_CHANGED + assert flags_changed == [CHANGING_FLAG_KEY] + + +def test_change_does_not_leak_into_the_next_scenario() -> None: + """Scenario isolation. + + A leak here would make the suite order-dependent: a scenario running after + the configuration-change one would start with ``changing-flag`` already + flipped, and the failure would look like a provider defect. + """ + control = InProcessControl() + + first = control.new_provider() + assert isinstance(first, ControllableInMemoryProvider) + baseline = _resolve_changing(first) + + control.change_flag() + assert _resolve_changing(first) != baseline, ( + "precondition: change_flag had no effect" + ) + + control.prepare_scenario() + + second = control.new_provider() + assert isinstance(second, ControllableInMemoryProvider) + assert _resolve_changing(second) == baseline, ( + "the next scenario did not start from the baseline" + ) + + +def test_change_flag_without_a_provider_fails_clearly() -> None: + """In-process the flag store and the provider are the same object, so there is + nothing to change before one exists. Saying so beats an AttributeError.""" + control = InProcessControl() + with pytest.raises(RuntimeError, match="must create one"): + control.change_flag() + + +def test_in_process_control_does_not_pretend_to_have_a_connection() -> None: + """The load-bearing one. + + A no-op ``disconnect`` would report the ``@stale`` scenarios as passed + against a provider that cannot go stale -- precisely the silent-green + failure a conformance suite must never have. ``InProcessControl`` therefore + does not implement ``ConnectionControl`` at all, and the TCK turns that into + a skip with a reason. + """ + assert not isinstance(InProcessControl(), ConnectionControl), ( + "InProcessControl implements ConnectionControl: an in-memory provider has no " + "connection to lose, and a no-op implementation would make the @stale " + "scenarios pass without testing anything" + ) + + +def test_canonical_flag_set_omits_missing_flag() -> None: + """The property the FLAG_NOT_FOUND scenario depends on. + + Seeding ``missing-flag`` would turn that scenario green for the wrong + reason, and nothing else in the suite would notice. + """ + assert "missing-flag" not in canonical_flag_set() + + +def test_update_flags_names_the_union_of_old_and_new_keys() -> None: + """Appendix A asks for the union, not just the new keys. + + A consumer caching evaluations needs to know everything that might have + changed, and a key that disappeared has changed as much as one that arrived. + """ + provider = ControllableInMemoryProvider(canonical_flag_set()) + + seen: list[list[str] | None] = [] + provider.attach(lambda _p, _e, details: seen.append(details.flags_changed)) + + provider.update_flags({}) + + assert seen, "no event was emitted" + assert seen[-1] is not None + assert set(seen[-1]) == set(canonical_flag_set()), ( + "the event did not name every flag that disappeared" + ) diff --git a/uv.lock b/uv.lock index 3168e251..b0ab379c 100644 --- a/uv.lock +++ b/uv.lock @@ -17,6 +17,7 @@ members = [ "openfeature-provider-flagd", "openfeature-provider-flipt", "openfeature-provider-ofrep", + "openfeature-provider-tck", "openfeature-provider-unleash", "openfeature-python-contrib", ] @@ -843,7 +844,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1868,7 +1869,7 @@ dev = [ [[package]] name = "openfeature-provider-flagd" -version = "0.5.1" +version = "0.5.2" source = { editable = "providers/openfeature-provider-flagd" } dependencies = [ { name = "cachebox" }, @@ -1989,6 +1990,37 @@ dev = [ { name = "types-requests", specifier = ">=2.32.0,<3.0.0" }, ] +[[package]] +name = "openfeature-provider-tck" +version = "0.1.0" +source = { editable = "tools/openfeature-provider-tck" } +dependencies = [ + { name = "openfeature-sdk" }, + { name = "pytest" }, + { name = "pytest-bdd" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage", extra = ["toml"] }, + { name = "mypy" }, + { name = "poethepoet" }, +] + +[package.metadata] +requires-dist = [ + { name = "openfeature-sdk", specifier = ">=0.8.2" }, + { name = "pytest", specifier = ">=8.4.0" }, + { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, + { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, + { name = "poethepoet", specifier = ">=0.37.0" }, +] + [[package]] name = "openfeature-provider-unleash" version = "0.1.2" @@ -2042,6 +2074,7 @@ dependencies = [ { name = "openfeature-provider-flagd" }, { name = "openfeature-provider-flipt" }, { name = "openfeature-provider-ofrep" }, + { name = "openfeature-provider-tck" }, { name = "openfeature-provider-unleash" }, ] @@ -2063,6 +2096,7 @@ requires-dist = [ { name = "openfeature-provider-flagd", editable = "providers/openfeature-provider-flagd" }, { name = "openfeature-provider-flipt", editable = "providers/openfeature-provider-flipt" }, { name = "openfeature-provider-ofrep", editable = "providers/openfeature-provider-ofrep" }, + { name = "openfeature-provider-tck", editable = "tools/openfeature-provider-tck" }, { name = "openfeature-provider-unleash", editable = "providers/openfeature-provider-unleash" }, ]