From 46bdf831d599450e08e5c5315f0592f82d7558db Mon Sep 17 00:00:00 2001 From: Manasjyoti Sharma Date: Sat, 27 Jun 2026 19:31:00 +0530 Subject: [PATCH] chore: prepare SDK for public repository launch --- .github/workflows/on-rc-tag.yml | 45 ++---- .github/workflows/publish-release.yml | 8 +- DEV.MD | 2 +- README.md | 143 +++++++++--------- docs/PROVIDERS.md | 13 +- pyproject.toml | 3 +- src/fortifyroot/_vendor/VENDOR_MANIFEST.json | 6 +- .../instrumentation/anthropic/__init__.py | 10 +- .../anthropic/retry_handler.py | 45 +++--- .../instrumentation/bedrock/__init__.py | 11 +- .../instrumentation/bedrock/retry_handler.py | 38 +++-- .../fortifyroot/retry_registry.py | 22 ++- .../instrumentation/langchain/__init__.py | 20 ++- .../langchain/callback_handler.py | 10 +- .../langchain/retry_handler.py | 43 +++--- .../instrumentation/langchain/span_utils.py | 2 +- .../instrumentation/litellm/__init__.py | 51 +++---- .../llamaindex/dispatcher_wrapper.py | 4 +- .../llamaindex/retry_handler.py | 21 ++- .../instrumentation/llamaindex/safety.py | 4 +- .../instrumentation/llamaindex/span_utils.py | 6 +- .../instrumentation/openai/retry_handler.py | 54 ++++--- .../openai/shared/chat_wrappers.py | 7 +- .../instrumentation/openai/v1/__init__.py | 4 +- src/fortifyroot/core.py | 24 ++- tests/conftest.py | 8 +- tests/openai/test_vcr.py | 8 +- tests/providers/conftest.py | 8 +- tests/test_env_mapping.py | 13 +- tests/test_init.py | 11 +- tests/test_metrics_temporality.py | 18 +-- tests/test_safety_runtime.py | 3 +- 32 files changed, 303 insertions(+), 362 deletions(-) diff --git a/.github/workflows/on-rc-tag.yml b/.github/workflows/on-rc-tag.yml index c96f2c4..2f44afa 100644 --- a/.github/workflows/on-rc-tag.yml +++ b/.github/workflows/on-rc-tag.yml @@ -1,17 +1,16 @@ name: SDK rc tag — dispatch Tier 2 -# Per RELEASE_GATE.md §2.4 + §4.4. Triggered by: +# Triggered by: # 1. Pushing an rc tag matching v-*-rc.* on this repo. # 2. workflow_dispatch with the rc_tag input (lets you re-run the gate -# after a manual dev-pin without cutting a new tag). +# without cutting a new tag). # # Body: -# 1. Read fr-backend's env/targets.yaml via the GitHub API. -# 2. Pre-flight: dev.image_tag must equal prod.image_tag (so the rc SDK -# is validated against the backend version customers actually run). +# 1. Read the backend release target manifest via the GitHub API. +# 2. Pre-flight: dev.image_tag must equal prod.image_tag. # If they differ, FAIL FAST with an actionable error message. -# 3. If they match (or prod is in `bootstrap` mode), dispatch -# event_type=sdk_rc to fr-system-tests with rc_tag in the payload. +# 3. If they match (or prod is in `bootstrap` mode), dispatch the SDK rc +# validation event with rc_tag in the payload. on: push: @@ -51,24 +50,20 @@ jobs: fi echo "rc_tag=$RC_TAG" >> "$GITHUB_OUTPUT" - - name: Read fr-backend env/targets.yaml + - name: Read backend release target manifest id: targets env: - # fr-backend is private. Keep this read-only token separate from - # the write-capable repository_dispatch token below. GH_TOKEN: ${{ secrets.BACKEND_TARGETS_READ_TOKEN }} run: | set -euo pipefail if [[ -z "${GH_TOKEN:-}" ]]; then - echo "::error::BACKEND_TARGETS_READ_TOKEN secret is required to read fr-backend env/targets.yaml." - echo "::error::Add a fine-grained PAT with Contents: Read-only on FortifyRoot/fr-backend." - echo "::error::See fr-meta/docs/CICD_SETUP.md §5.3." + echo "::error::Repository secret BACKEND_TARGETS_READ_TOKEN is not configured." exit 1 fi - # Read from `main` because that is the GitOps source of truth. + # Read from `main` because that is the release target source of truth. if ! gh api --method GET repos/FortifyRoot/fr-backend/contents/env/targets.yaml \ -f ref=main --jq '.content' | base64 -d > targets.yaml; then - echo "::error::Could not fetch FortifyRoot/fr-backend env/targets.yaml from main" + echo "::error::Could not fetch the backend release target manifest from main." exit 1 fi # `yq` (mikefarah/yq) is preinstalled on GitHub-hosted runners. @@ -97,30 +92,22 @@ jobs: dev.image_tag = $DEV prod.image_tag = $PROD Dev must match prod for SDK rc validation. To fix: - 1. Open a PR on fr-backend that sets env/targets.yaml: - dev.image_tag = $PROD - 2. Merge it. deploy-dev.yml will roll dev-api back to '$PROD' - and run Tier 1. - 3. Wait for Tier 1 green, then re-run THIS workflow via - workflow_dispatch with rc_tag = ${{ steps.vars.outputs.rc_tag }}. + 1. Align the backend dev target with prod.image_tag ($PROD). + 2. Wait for the dev deployment and validation gate to complete. + 3. Re-run this workflow with rc_tag = ${{ steps.vars.outputs.rc_tag }}. EOF exit 1 fi echo "Pre-flight OK — dev == prod == $DEV" - - name: Dispatch sdk_rc to fr-system-tests + - name: Dispatch SDK rc validation env: - # SYSTEM_TEST_DISPATCH_TOKEN is REQUIRED for cross-repo dispatch. - # GITHUB_TOKEN cannot fire `repository_dispatch` on a foreign repo. - # See CICD_SETUP.md §5.3. GH_TOKEN: ${{ secrets.SYSTEM_TEST_DISPATCH_TOKEN }} RC_TAG: ${{ steps.vars.outputs.rc_tag }} run: | set -euo pipefail if [[ -z "${GH_TOKEN:-}" ]]; then - echo "::error::SYSTEM_TEST_DISPATCH_TOKEN secret is required to dispatch sdk_rc to fr-system-tests." - echo "::error::Add a fine-grained PAT with Contents: Read and write on FortifyRoot/fr-system-tests." - echo "::error::See fr-meta/docs/CICD_SETUP.md §5.3." + echo "::error::Repository secret SYSTEM_TEST_DISPATCH_TOKEN is not configured." exit 1 fi gh api -X POST \ @@ -128,4 +115,4 @@ jobs: -f event_type=sdk_rc \ -F "client_payload[rc_tag]=${RC_TAG}" \ -F "client_payload[repo]=${{ github.repository }}" - echo "sdk_rc dispatched. Watch fr-system-tests Actions for result." + echo "SDK rc validation dispatched." diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 1739d6c..07a9bc9 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -1,9 +1,6 @@ name: Publish to PyPI -# Manual workflow_dispatch only — see RELEASE_GATE.md §9.3. -# -# Hardening status: Tier 1 items 1-3 implemented; T1.4 + Tier 2 + Tier 3 -# tracked in fr-system-tests/docs/SDK_PUBLISH_HARDENING.md. +# Manual workflow_dispatch only. # # Body (in safe order — no user-supplied code runs until ALL pre-flight # checks pass): @@ -37,7 +34,8 @@ jobs: name: Publish ${{ inputs.rc_tag }} to ${{ inputs.pypi_repository }} runs-on: ubuntu-latest timeout-minutes: 30 - environment: pypi # 1 reviewer (the releaser) per RELEASE_GATE.md §7 L2-15 + # Requires the configured PyPI release environment approval. + environment: pypi steps: # --------------------------------------------------------------- # T1.2 — Validate the rc_tag input matches a strict pattern. diff --git a/DEV.MD b/DEV.MD index b60fff2..a2dfa10 100644 --- a/DEV.MD +++ b/DEV.MD @@ -5,7 +5,7 @@ It describes how to set up a local development environment, manage the FortifyRoot fork of OpenLLMetry (Traceloop), vendor OpenLLMetry into the FortifyRoot Ocelle SDK, and work effectively with Poetry and VS Code. -This file is internal-facing and intentionally separate from README.md. +This file is maintainer-facing and intentionally separate from README.md. --- diff --git a/README.md b/README.md index fb18016..b7c122f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,16 @@ # FortifyRoot Ocelle -FortifyRoot Ocelle is a Python SDK for LLM observability, safety, and auditability. With a single initialization call, Ocelle instruments supported LLM providers and frameworks, captures traces, records token/latency metadata, and applies FortifyRoot safety callbacks for prompt and completion content. +FortifyRoot Ocelle is the Python SDK for FortifyRoot LLM observability and runtime safety. Add one initialization call to your application and Ocelle will instrument supported LLM providers and frameworks, emit OpenTelemetry traces/metrics/logs to FortifyRoot, and apply configured prompt/completion safety rules before sensitive content leaves or enters your application flow. + +Ocelle is built on a FortifyRoot-maintained OpenLLMetry fork, vendored into this repository for dependency isolation, safety extensions, and stable FortifyRoot packaging. + +## What Ocelle Captures + +- LLM request/response traces with provider, model, span hierarchy, token usage, latency, retry-attempt, and streaming timing metadata. +- Framework spans for workflows, tasks, agents, and tools through decorators and supported framework integrations. +- Optional prompt and completion content when content tracing is enabled. +- Safety findings for prompt and completion content, including masking decisions and rule metadata. +- OTLP traces, metrics, and correlated logs for FortifyRoot ingestion. ## Installation @@ -10,11 +20,12 @@ Ocelle supports Python 3.10 and newer. pip install fortifyroot-ocelle ``` -Install provider extras as needed: +Install only the provider/framework extras your application uses: ```bash pip install "fortifyroot-ocelle[openai]" pip install "fortifyroot-ocelle[openai,anthropic,langchain]" +pip install "fortifyroot-ocelle[bedrock,litellm,llamaindex]" ``` ## Quick Start @@ -23,20 +34,21 @@ pip install "fortifyroot-ocelle[openai,anthropic,langchain]" import fortifyroot.ocelle as ocelle ocelle.init( - app_name="my-llm-app", # any name you choose for this service + app_name="my-llm-app", api_key="fr_sk_...", - resource_attributes={"environment": "dev"}, # dev / prod / testing — or any label you choose + resource_attributes={"environment": "prod"}, ) -import openai +from openai import OpenAI -response = openai.chat.completions.create( - model="gpt-4", +client = OpenAI() +response = client.chat.completions.create( + model="gpt-4o-mini", messages=[{"role": "user", "content": "Hello!"}], ) ``` -The canonical import is: +The canonical SDK import is: ```python import fortifyroot.ocelle as ocelle @@ -48,55 +60,49 @@ The package also exposes a convenience alias: import ocelle ``` -The root `fortifyroot` package is reserved for internal namespaces such as vendored instrumentation. Public SDK code should use `fortifyroot.ocelle` or the `ocelle` convenience alias. +## Supported Providers And Frameworks -## Network Requirements +This table lists the launch-supported instrumentation exposed through `fortifyroot.ocelle.Instruments` and SDK extras. Package ranges are the ranges declared by this SDK; they are not a claim about the latest upstream release. -If your app runs in a private subnet, VPC, Kubernetes cluster, or locked-down CI/runtime, allow outbound HTTPS egress on TCP 443 to `api.fortifyroot.com`. +| Library / framework | Instrument | Extra | Declared package range | Sync | Async | Streaming | Safety | +|---|---|---|---|---:|---:|---:|---| +| OpenAI | `Instruments.OPENAI` | `openai` | `openai >=1.31.1,<3` | Yes | Yes | Yes | Prompt + completion, including streaming paths | +| Anthropic | `Instruments.ANTHROPIC` | `anthropic` | `anthropic >=0.49,<1.0.0` | Yes | Yes | Yes | Prompt + completion, including streaming paths | +| Google GenAI / Gemini | `Instruments.GOOGLE_GENERATIVEAI` | `google-generativeai` | `google-genai >=1.0.0,<2` | Yes | Yes | Yes | Prompt + completion, including streaming paths | +| AWS Bedrock Runtime | `Instruments.BEDROCK` | `bedrock` | `boto3 >=1.34.120,<2` | Yes | No native async client path | Yes | Prompt + completion for invoke/converse paths, including stream wrappers | +| LiteLLM | `Instruments.LITELLM` | `litellm` | `litellm >=1.71.2,<2,!=1.82.7,!=1.82.8` | Yes | Yes | Yes | Prompt + completion, including streaming paths | +| LangChain | `Instruments.LANGCHAIN` | `langchain` | `langchain >=0.2.5,<2.0.0`, `langchain-openai >=0.1.15,<2.0.0` | Yes | Yes | Provider-dependent | Prompt + completion for supported chat/LLM paths | +| LangGraph | via `Instruments.LANGCHAIN` | install with app | Covered through LangChain/OpenAI launch path | Yes | Yes | Provider-dependent | Same supported path as LangChain | +| LlamaIndex | `Instruments.LLAMA_INDEX` | `llamaindex` | `llama-index >=0.14.12,<0.15.0` | Yes | Yes | Yes | Prompt + completion, including streaming paths | -Ocelle exports telemetry over OTLP/HTTP to: +For provider-role behavior, routed providers such as OpenRouter, LiteLLM, Bedrock, Azure OpenAI, and planned/mapper-supported providers, see [Provider Support](docs/PROVIDERS.md). That document is the source of truth for what is launch-certified versus planned. -- `https://api.fortifyroot.com/v1/traces` -- `https://api.fortifyroot.com/v1/metrics` -- `https://api.fortifyroot.com/v1/logs` - -If safety enforcement is enabled with `config_profile_id`, the SDK also polls: - -- `https://api.fortifyroot.com/v1/sdk/config/{config_profile_id}` +## Runtime Safety -No inbound firewall rule is required. Hosted FortifyRoot usage does not require opening OTLP ports `4317` or `4318`; those are local/internal listener ports. Your workload still needs separate egress to whichever LLM providers it calls. +Ocelle can poll a FortifyRoot SDK config profile and apply configured safety rules locally in the SDK. Rules can inspect prompt and completion text and currently resolve to `ALLOW` or `MASK`. -## Auto-Instrumented LLM Libraries +Supported safety categories are: -The MVP SDK vendors and exposes the following supported instrumentation packages: +`PII`, `PCI`, `PHI`, `API_KEY`, `SECRET`, `PROMPT_INJECTION`, `PROFANITY`, `TOXICITY`, `VIOLENCE`, `SELF_HARM`, `CONFIDENTIAL`, and `CUSTOM`. -- **LLM providers**: OpenAI, Anthropic, Google Generative AI, AWS Bedrock, LiteLLM -- **Frameworks**: LangChain, LlamaIndex +Rules can be backed by regex matchers, string-list matchers, or approved user-defined detectors. Masking is applied before the instrumented provider/framework returns the text to application code where the integration can safely mutate the response object or stream chunk. -For the current launch-certified provider-role matrix and support tiers, see [Provider Support](docs/PROVIDERS.md). +Because Ocelle is open source, the SDK's enforcement flow is visible by design. Organization-specific safety policy is fetched at runtime from your FortifyRoot SDK config profile, so public code review exposes the engine and built-in defaults, not customer-specific rules. ## Configuration ### Environment Variables -Ocelle keeps the FortifyRoot environment variable namespace stable: - -| Environment Variable | Description | Default | -|---------------------|-------------|---------| +| Environment variable | Description | Default | +|---|---|---| | `FORTIFYROOT_API_KEY` | FortifyRoot API key | None | -| `FORTIFYROOT_BASE_URL` | API endpoint URL | `https://api.fortifyroot.com` | +| `FORTIFYROOT_BASE_URL` | FortifyRoot API endpoint | `https://api.fortifyroot.com` | | `FORTIFYROOT_TRACE_CONTENT` | Capture prompt/response content | `true` | -| `FORTIFYROOT_TRACING_ENABLED` | Enable/disable tracing | `true` | -| `FORTIFYROOT_METRICS_ENABLED` | Enable/disable metrics | `true` | +| `FORTIFYROOT_TRACING_ENABLED` | Enable trace export | `true` | +| `FORTIFYROOT_METRICS_ENABLED` | Enable metric export | `true` | | `FORTIFYROOT_LOGGING_ENABLED` | Enable OTLP log export and synthetic span-end logs | `false` | -When `FORTIFYROOT_LOGGING_ENABLED=true`: - -- stdlib Python `logging` records keep using the app's existing handlers and formatting -- stdlib Python `logging` records emitted inside active spans are exported with trace/span correlation -- stdlib Python `logging` records emitted outside active spans can still be exported, but they remain uncorrelated -- Ocelle emits one synthetic correlated log for each completed instrumented span -- `print(...)`, stdout, and stderr capture are not included in MVP; use Python `logging` for application logs +When `FORTIFYROOT_LOGGING_ENABLED=true`, Python `logging` records emitted inside active spans are exported with trace/span correlation. `print(...)`, stdout, and stderr capture are not included; use Python `logging` for application logs. ### Programmatic Configuration @@ -127,38 +133,9 @@ ocelle.configure() \ .init() ``` -### Advanced Configuration - -```python -import fortifyroot.ocelle as ocelle -from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor -from opentelemetry.sdk.trace.sampling import TraceIdRatioBased - -ocelle.init( - app_name="my-app", - api_key="fr_sk_...", - sampler=TraceIdRatioBased(0.1), -) - -ocelle.init( - app_name="my-app", - processors=[SimpleSpanProcessor(ConsoleSpanExporter())], -) - -def span_callback(span): - # Inspect span attributes, log alerts, etc. - pass - -ocelle.init( - app_name="my-app", - api_key="fr_sk_...", - span_postprocess_callback=span_callback, -) -``` - ## Decorators -Use decorators to trace custom functions and create hierarchical traces: +Use decorators to add trace structure around your own application logic: ```python from fortifyroot.ocelle import agent, task, tool, workflow @@ -184,7 +161,7 @@ def generate_answer(context, question): ## Association Properties -Attach custom properties to traces for filtering and correlation: +Attach properties to traces for filtering and correlation: ```python import fortifyroot.ocelle as ocelle @@ -198,9 +175,25 @@ ocelle.set_association_properties({ }) ``` +## Network Requirements + +If your app runs in a private subnet, VPC, Kubernetes cluster, or locked-down runtime, allow outbound HTTPS egress on TCP 443 to `api.fortifyroot.com`. + +Ocelle exports telemetry over OTLP/HTTP to: + +- `https://api.fortifyroot.com/v1/traces` +- `https://api.fortifyroot.com/v1/metrics` +- `https://api.fortifyroot.com/v1/logs` + +If safety enforcement is enabled with `config_profile_id`, the SDK also polls: + +- `https://api.fortifyroot.com/v1/sdk/config/{config_profile_id}` + +No inbound firewall rule is required. Hosted FortifyRoot usage does not require opening OTLP ports `4317` or `4318`. Your workload still needs separate egress to whichever LLM providers it calls. + ## Privacy And Content Tracing -To disable prompt and response content capture: +Disable prompt and response content capture with: ```bash export FORTIFYROOT_TRACE_CONTENT=false @@ -218,10 +211,10 @@ ocelle.init( ) ``` -## Attribution - -FortifyRoot Ocelle includes code derived from [OpenLLMetry](https://github.com/traceloop/openllmetry) and `traceloop-sdk` by Traceloop, licensed under the Apache License, Version 2.0. The SDK retains the Apache 2.0 license text and attribution in [LICENSE](LICENSE). +Safety rules can still run when configured; content tracing controls what is exported as telemetry content. -## License +## License And Attribution Apache License, Version 2.0. + +FortifyRoot Ocelle includes code derived from [OpenLLMetry](https://github.com/traceloop/openllmetry) and `traceloop-sdk` by Traceloop, licensed under the Apache License, Version 2.0. The license text and attribution are retained in [LICENSE](LICENSE). diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index 47a74c7..dd0e120 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -1,14 +1,15 @@ # FortifyRoot Provider Support -This page is the current launch-certified support matrix for FortifyRoot Ocelle SDK provider and framework instrumentation. +This page is the current support matrix for FortifyRoot Ocelle SDK provider and framework instrumentation. -The MVP SDK vendors and exposes only launch-supported instrumentation packages. Planned rows below document roadmap/provider-role direction, not libraries currently exposed by `fortifyroot.ocelle.Instruments` or SDK extras. +The MVP SDK vendors and exposes only launch-supported instrumentation packages. Planned and mapper-supported rows below document roadmap/provider-role direction, not libraries currently exposed by `fortifyroot.ocelle.Instruments` or SDK extras. ## Status Terms | Status | Meaning | |---|---| | `live-tested` | Verified with live provider traffic in the launch validation suite. | +| `opt-in-live-tested` | Covered by an opt-in live-dev test, but not part of the default fixture-mode launch gate. | | `fixture-tested` | Covered by recorded or deterministic test fixtures, but not claimed as live-tested in the launch matrix. | | `mapper-supported` | FortifyRoot recognizes the provider role, but full live coverage is not part of the current launch matrix. | | `planned` | Planned or broader-direction support; do not describe as launch-certified. | @@ -32,8 +33,8 @@ Raw SDK/request values can differ from these roles. FortifyRoot keeps raw values | OpenAI | `live-tested` | Yes | Direct calls: none | Direct calls: `openai` | OpenAI-compatible routes can change routing/billing provider; see OpenRouter. | | Anthropic | `live-tested` | Yes | Direct calls: none | Direct calls: `anthropic` | Also appears as model provider for Claude models routed through OpenRouter or LiteLLM. | | Google GenAI | `live-tested` | Yes | Direct calls: none | Direct calls: `google` | Launch validation covers Gemini-style traffic. | -| xAI | `live-tested` | Yes | Direct calls: none in current launch matrix | Direct calls: `xai`; OpenRouter-routed calls: `openrouter` | Recognized through OpenAI-compatible/OpenRouter paths and provider-role mapping; not a dedicated vendored instrument. | -| AWS Bedrock | `live-tested` | Yes | `bedrock` where Bedrock is the platform route | `bedrock` where Bedrock pricing applies | Includes Bedrock-native and Bedrock-routed provider-role behavior in the launch validation path. | +| xAI | `mapper-supported` | Yes, when inferred from model names or routed OpenAI-compatible telemetry | Direct calls: none in current launch matrix | Inferred from provider-role mapping when deterministic | Recognized through OpenAI-compatible/OpenRouter paths and provider-role mapping; not a dedicated vendored instrument. | +| AWS Bedrock | `live-tested` | Yes | `bedrock` where Bedrock is the platform route | `bedrock` where Bedrock pricing applies | Includes Bedrock-native and third-party-on-Bedrock provider-role behavior in the launch validation path. | | Azure OpenAI | `mapper-supported` | Usually `openai` | `azure` | `azure` where Azure pricing applies | Recognized through the OpenAI-compatible instrumentation path and provider-role mapping; not a dedicated vendored instrument. | | Cohere | `planned` | Planned | Direct calls: none | Provider pricing when certified | Not bundled or exposed in the MVP SDK. | | Mistral AI | `planned` | Planned | Direct calls: none | Provider pricing when certified | Not bundled or exposed in the MVP SDK. | @@ -51,7 +52,7 @@ Raw SDK/request values can differ from these roles. FortifyRoot keeps raw values | Provider | Current status | Routing provider behavior | Billing provider behavior | Notes | |---|---:|---|---|---| -| OpenRouter | `live-tested` | `routing_provider=openrouter` when traffic is sent through OpenRouter. | `billing_provider=openrouter` when the route is identified and pricing data is available. | Model provider remains the underlying vendor, such as `anthropic`, `openai`, `google`, or `xai`. | +| OpenRouter | `opt-in-live-tested` | `routing_provider=openrouter` when traffic is sent through OpenRouter and the route is detected. | `billing_provider=openrouter` when the route is identified and pricing data is available. | Model provider remains the underlying vendor, such as `anthropic`, `openai`, `google`, or `xai`. Routed OpenRouter cost checks are opt-in live-dev assertions, not part of the default fixture-mode gate. | | LiteLLM self-hosted | `live-tested` | `routing_provider=litellm`. | Defaults to the underlying model provider unless another routed billing provider is explicitly identified. | FortifyRoot does not infer LiteLLM Cloud billing from self-hosted LiteLLM traffic. | | LiteLLM Cloud | `planned` | Needs an explicit Cloud/self-hosted signal. | Planned; requires LiteLLM Cloud pricing and an org/config signal before `billing_provider=litellm` is emitted. | Do not describe LiteLLM Cloud billing as launch-certified. | | Azure | `mapper-supported` | `routing_provider=azure` for Azure-routed OpenAI traffic. | `billing_provider=azure` where Azure pricing applies. | Supported by provider-role mapping; validate live for launch claims. | @@ -77,7 +78,7 @@ Vector database instrumentation is separate from LLM provider-role attribution a ## Caveats -- This page is a launch-certified support matrix, not an exhaustive list of every library that can be imported or partially instrumented. +- This page is a support matrix, not an exhaustive list of every library that can be imported or partially instrumented. - Provider-role behavior requires telemetry emitted by a current SDK and FortifyRoot service version. Older telemetry can lack newer role labels. - Pricing support depends on available pricing data and deterministic billing-provider identification. - LiteLLM Cloud billing is planned, not launch-certified. diff --git a/pyproject.toml b/pyproject.toml index 85d2d59..a278b9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,8 @@ wrapt = "^1.14.0" # TracerWrapper's auto-load path imports it even when the customer did # not install the ``[llamaindex]`` extra. Keeping it optional caused the # instrumentor to silently swallow ImportError and emit NO LlamaIndex -# spans for the customer (see fr-system-tests ST-6 addendum 2, I2). +# spans for the customer; keep it as a hard dependency while the vendored +# LlamaIndex instrumentor is auto-loaded. # It's a 40 KB pure-Python package — cheap to always install. inflection = ">=0.5.1,<0.6.0" diff --git a/src/fortifyroot/_vendor/VENDOR_MANIFEST.json b/src/fortifyroot/_vendor/VENDOR_MANIFEST.json index a6d6dc2..15712fd 100644 --- a/src/fortifyroot/_vendor/VENDOR_MANIFEST.json +++ b/src/fortifyroot/_vendor/VENDOR_MANIFEST.json @@ -1,9 +1,9 @@ { - "vendored_at": "2026-06-20T23:35:14.965076", + "vendored_at": "2026-06-27T19:30:27.492730", "openllmetry_version": "0.52.6", - "git_commit": "0d03c3f8fba0", + "git_commit": "86389b2e0d35", "git_branch": "HEAD", - "git_tag": "fr-v0.52.6.32", + "git_tag": "fr-v0.52.6.33", "instrumentation_package_policy": { "opentelemetry-instrumentation-agno": false, "opentelemetry-instrumentation-alephalpha": false, diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/anthropic/__init__.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/anthropic/__init__.py index 46ce813..a8c30fe 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/anthropic/__init__.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/anthropic/__init__.py @@ -557,7 +557,7 @@ def _wrap( kwargs = _apply_prompt_safety(span, kwargs, name) _handle_input(span, event_logger, kwargs) start_time = time.perf_counter() - # ST-10.4 (review-driven 2026-05-17): make the anthropic.chat span + # Make the anthropic.chat span # the AMBIENT OTel context for the duration of the SDK call so the # FortifyRoot retry handler (wrapping # ``anthropic._base_client.SyncHttpxClientWrapper.send``) can @@ -698,7 +698,7 @@ async def _awrap( kwargs = await asyncio.to_thread(_apply_prompt_safety, span, kwargs, name) # FR: async safety await _ahandle_input(span, event_logger, kwargs) start_time = time.perf_counter() - # ST-10.4 (review-driven 2026-05-17): see sync _wrap above for + # See sync _wrap above for # rationale on use_span(end_on_exit=False) around the wrapped call. try: with trace.use_span(span, end_on_exit=False): @@ -898,16 +898,16 @@ def _instrument(self, **kwargs): except Exception: pass # that's ok, we don't want to fail if some methods do not exist - # ST-10.4: per-attempt retry_attempt emission via private + # Per-attempt retry_attempt emission via private # ``anthropic._base_client`` httpx wrapper classes. Guarded # against missing private symbols (logs warning + skips emission). # Pass the same tracer_provider the rest of the instrumentor # uses so retry_attempt spans land in the same exporter as the - # anthropic logical span (review-driven 2026-05-16 fix). + # anthropic logical span. instrument_retry_emitter(tracer_provider=tracer_provider) def _uninstrument(self, **kwargs): - uninstrument_retry_emitter() # ST-10.4 symmetry + uninstrument_retry_emitter() # retry-emitter symmetry for wrapped_method in WRAPPED_METHODS: wrap_package = wrapped_method.get("package") wrap_object = wrapped_method.get("object") diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/anthropic/retry_handler.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/anthropic/retry_handler.py index 0151e5d..bc18e2c 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/anthropic/retry_handler.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/anthropic/retry_handler.py @@ -1,7 +1,6 @@ -"""ST-10.4 retry-aware emission for the Anthropic direct SDK. +"""Retry-aware emission for the Anthropic direct SDK. -Per RETRY_LOOP.md §4.4 Anthropic row + §4.7 suppression discipline + -ST-10.0 hook-table addendum (in phase_st10_retryloop.txt): +Design contract: - Hook ``anthropic._base_client.SyncHttpxClientWrapper.send`` and ``AsyncHttpxClientWrapper.send`` — fires once per HTTP attempt @@ -16,17 +15,16 @@ - Endpoint allow-list: only LLM endpoints emit retry_attempt spans (``/v1/messages`` for the modern Messages API, ``/v1/complete`` for legacy completions). Anything else (token - refresh, model listing, etc.) does NOT emit retry_attempt — per - §4.4.1 allow-listing requirement. - - Suppression discipline (§4.7): check BOTH the OTel context + refresh, model listing, etc.) does NOT emit retry_attempt. + - Suppression discipline: check BOTH the OTel context ``SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY`` AND the shared ``is_framework_owned()`` registry. If either says "suppress", skip emission. Prevents framework retries (LiteLLM / LangChain / LlamaIndex) from DOUBLE-emitting when both framework wrappers and direct-SDK wrappers are active. - Parent resolution: use the current OTel ambient span. If invalid - or absent, skip gracefully (no orphan retry_attempt span). The - §4.5 backend dedup degrades gracefully. + or absent, skip gracefully (no orphan retry_attempt span). Backend + dedup degrades gracefully. Tests: see ``tests/test_retry_attempt_emission.py``. """ @@ -54,7 +52,7 @@ logger = logging.getLogger(__name__) -# ST-10 §4.4 / §4.5 constants. +# Retry-attempt constants. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.anthropic" _FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" @@ -199,9 +197,7 @@ def _start_attempt_span(request: Any, parent_span: "trace.Span") -> "trace.Span" # ``anthropic/__init__.py`` ``_wrap``). The earlier draft of # this handler used lower-case "anthropic" for cross-handler # consistency, but that mismatched the upstream Anthropic - # instrumentor and the fr-system-tests ProviderModel - # ``gen_ai_system="Anthropic"`` assertion — ST-10.4 - # review-driven fix 2026-05-17. Backend provider grouping is + # instrumentor and provider-model assertions. Backend provider grouping is # case-sensitive in practice; canonicalisation happens via # ``event_provider`` (which is always lower-case in tests). "gen_ai.system": "Anthropic", @@ -229,11 +225,11 @@ def _finalize_success(span: "trace.Span", response: Any, *, is_streaming: bool = For non-streaming responses (regardless of 2xx vs non-2xx), parse the JSON body and extract usage tokens + response id + response - model. Per RETRY_LOOP.md §4.4 token-usage rule (around line 164): + model. Per the retry-attempt token-usage rule: wrappers MUST extract usage from the response body whenever it's present, regardless of whether the attempt succeeded — some failures DO consume tokens and the provider returns usage in the - error body. §4.5 backend dedup makes the qualifying retry_attempt + error body. Backend dedup makes the qualifying retry_attempt canonical (even single-attempt), so usage must live on this span. Streaming responses (``stream=True`` passed to ``send()``) skip @@ -257,7 +253,7 @@ def _finalize_success(span: "trace.Span", response: Any, *, is_streaming: bool = pass # Non-streaming body parse — applies to BOTH 2xx and non-2xx per the - # §4.4 token-usage rule. ``_extract_usage_from_body`` is fully + # retry-attempt token-usage rule. ``_extract_usage_from_body`` is fully # defensive (missing fields / parse failure / no ``usage`` block all # degrade silently). if not is_streaming: @@ -348,22 +344,21 @@ def _should_emit_for(request: Any) -> bool: def _sync_send_wrapper(wrapped, instance, args, kwargs): """Wraps ``anthropic._base_client.SyncHttpxClientWrapper.send``. - Direct-SDK wrappers do NOT register tokens in the §4.7.1 framework + Direct-SDK wrappers do NOT register tokens in the framework-attempt registry (the registry's contract reserves registration for FRAMEWORK wrappers — LiteLLM / LangChain / LlamaIndex). They only consult via ``is_framework_owned()``. Self-registering would falsely suppress concurrent direct-SDK calls on the same OS thread, which manifests most visibly under asyncio (multiple tasks sharing one thread). - Streaming skip (ST-10.4 review-driven 2026-05-17): when + Streaming skip: when ``stream=True`` is passed to send, this wrapper SKIPS retry_attempt emission. Streaming retry_attempts cannot carry usage (SSE stream - can't be peeked) but §4.5 dedup would still promote them to the + can't be peeked) but backend dedup would still promote them to the canonical LLMUsageEvent, producing zero-token events. Leaving the parent ``anthropic.chat`` span as the canonical (it gets full usage from the Anthropic streaming wrapper). Streaming retry-loop - detection is the deferred follow-up - ``ST-10.4-FOLLOWUP-streaming-usage``. + detection is a deferred streaming-usage follow-up. """ request = args[0] if args else kwargs.get("request") if request is None or not _should_emit_for(request): @@ -490,7 +485,7 @@ def instrument_retry_emitter(tracer_provider=None) -> None: ) if not sync_ok: logger.warning( - "ST-10.4: anthropic._base_client.%s.%s missing/incompatible; " + "Anthropic retry emitter: anthropic._base_client.%s.%s missing/incompatible; " "skipping sync retry_attempt emission. Normal anthropic " "instrumentation is unaffected.", _SYNC_WRAPPER_CLASS, _WRAPPED_METHOD, @@ -504,13 +499,13 @@ def instrument_retry_emitter(tracer_provider=None) -> None: ) except Exception as e: logger.warning( - "ST-10.4: failed to wrap anthropic sync httpx send (%s); " + "Anthropic retry emitter: failed to wrap anthropic sync httpx send (%s); " "retry_attempt emission disabled for sync path", e, ) if not async_ok: logger.warning( - "ST-10.4: anthropic._base_client.%s.%s missing/incompatible; " + "Anthropic retry emitter: anthropic._base_client.%s.%s missing/incompatible; " "skipping async retry_attempt emission. Normal anthropic " "instrumentation is unaffected.", _ASYNC_WRAPPER_CLASS, _WRAPPED_METHOD, @@ -524,7 +519,7 @@ def instrument_retry_emitter(tracer_provider=None) -> None: ) except Exception as e: logger.warning( - "ST-10.4: failed to wrap anthropic async httpx send (%s); " + "Anthropic retry emitter: failed to wrap anthropic async httpx send (%s); " "retry_attempt emission disabled for async path", e, ) @@ -543,7 +538,7 @@ def uninstrument_retry_emitter() -> None: unwrap(f"{_ANTHROPIC_BASE_CLIENT_MODULE}.{cls}", _WRAPPED_METHOD) except Exception: logger.debug( - "ST-10.4: anthropic unwrap of %s.%s failed (likely " + "Anthropic retry emitter: unwrap of %s.%s failed (likely " "wrap was never installed for this variant)", cls, _WRAPPED_METHOD, exc_info=True, diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/bedrock/__init__.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/bedrock/__init__.py index 5945ffa..ac0bdf9 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/bedrock/__init__.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/bedrock/__init__.py @@ -131,8 +131,7 @@ def is_metrics_enabled() -> bool: def _with_tracer_wrapper(func): """Helper for providing tracer for wrapper functions. - ``tracer_provider`` (added 2026-05-16 for the ST-10.4 review-driven - fix) is plumbed alongside the tracer so the bedrock-runtime client + ``tracer_provider`` is plumbed alongside the tracer so the bedrock-runtime client wrap can pass it to ``install_event_hooks_on_client`` — letting the retry handler's event hooks emit spans through the same provider the rest of the instrumentor uses. @@ -201,12 +200,12 @@ def _wrap( client.converse_stream = _instrumented_converse_stream( client.converse_stream, tracer, metric_params, event_logger ) - # ST-10.4: register per-attempt botocore event hooks on the + # Register per-attempt botocore event hooks on the # bedrock-runtime client. Public botocore API; emits one # retry_attempt sibling span per HTTP attempt under the outer # bedrock.completion / bedrock.converse span. ``tracer_provider`` # is passed so retry_attempt spans go to the same provider as - # the bedrock logical span — review-driven 2026-05-16 fix. + # the bedrock logical span. install_event_hooks_on_client(client, tracer_provider=tracer_provider) return client except Exception as e: @@ -254,7 +253,7 @@ def with_instrumentation(*args, **kwargs): span = tracer.start_span(_BEDROCK_INVOKE_SPAN_NAME, kind=SpanKind.CLIENT) - # ST-10.4: make the streaming span the AMBIENT OTel context for + # Make the streaming span the AMBIENT OTel context for # the duration of the underlying boto3 call so per-attempt # botocore event hooks (before-send.bedrock-runtime.*) can # resolve this span as the retry_attempt parent. The span is @@ -304,7 +303,7 @@ def with_instrumentation(*args, **kwargs): span = tracer.start_span(_BEDROCK_CONVERSE_SPAN_NAME, kind=SpanKind.CLIENT) kwargs = _apply_converse_prompt_safety(span, kwargs, _BEDROCK_CONVERSE_SPAN_NAME) - # ST-10.4: see _instrumented_model_invoke_with_response_stream + # See _instrumented_model_invoke_with_response_stream # for the rationale on use_span(end_on_exit=False). stream_start_time = time.perf_counter() with trace.use_span(span, end_on_exit=False): diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/bedrock/retry_handler.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/bedrock/retry_handler.py index 7a1d4c1..ad4a588 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/bedrock/retry_handler.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/bedrock/retry_handler.py @@ -1,7 +1,6 @@ -"""ST-10.4 retry-aware emission for the Bedrock (botocore) direct SDK. +"""Retry-aware emission for the Bedrock (botocore) direct SDK. -Per RETRY_LOOP.md §4.4 Bedrock row + §4.7 suppression discipline + -ST-10.0 hook-table addendum (in phase_st10_retryloop.txt): +Design contract: - Use botocore's PUBLIC event-hook API on bedrock-runtime clients: * ``before-send.bedrock-runtime.*`` fires once per HTTP attempt @@ -23,14 +22,13 @@ do NOT register a framework-attempt token here — the §4.7.1 registry's documented contract reserves registration for FRAMEWORK wrappers (LiteLLM / LangChain / LlamaIndex). Direct-SDK wrappers - only CONSULT via ``is_framework_owned()``. (See the 2026-05-13 - review-driven C1 fix; an earlier draft of this module incorrectly - registered a token.) + only CONSULT via ``is_framework_owned()``. An earlier draft of + this module incorrectly registered a token. - Endpoint allow-list: implicit via the event-name pattern ``bedrock-runtime.*``. Only bedrock-runtime operations (Invoke / Converse and their streaming variants) fire these events; non-LLM AWS service traffic (S3, STS, etc.) is unaffected. - - Suppression discipline (§4.7): before emitting, check BOTH the + - Suppression discipline: before emitting, check BOTH the OTel context ``SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY`` AND the shared ``is_framework_owned()`` registry (consult-only — see above). @@ -69,7 +67,7 @@ logger = logging.getLogger(__name__) -# ST-10 §4.4 / §4.5 constants. +# Retry-attempt constants. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.bedrock" _FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" @@ -93,7 +91,7 @@ # the parent span on their provider but the retry_attempt span sent # into a no-op tracer. Module-level (not per-client) because all # bedrock-runtime clients created under one instrumentor share the -# same provider config. (review-driven 2026-05-16 fix.) +# same provider config. _tracer_provider = None @@ -309,14 +307,14 @@ def _before_send_hook(event_name: Optional[str] = None, request: Any = None, **_ ``request.context`` for retrieval by the paired ``response-received`` hook. - No self-registration in the §4.7.1 framework registry: the registry's + No self-registration in the framework-attempt registry: the registry's contract reserves registration for FRAMEWORK wrappers (LiteLLM / LangChain / LlamaIndex). Direct-SDK wrappers only CONSULT via ``is_framework_owned()``. Defensive cleanup of any prior context-stored span on this request still runs, but it no longer touches the framework registry. - Streaming skip (ST-10.4 review-driven 2026-05-17): when the + Streaming skip: when the botocore operation is a streaming one (event name ends with ``Stream``: ``InvokeModelWithResponseStream`` / ``ConverseStream``), this hook skips retry_attempt emission. @@ -324,12 +322,11 @@ def _before_send_hook(event_name: Optional[str] = None, request: Any = None, **_ cannot carry token usage at attempt-end (usage arrives via the stream-completion callback installed by the Bedrock streaming wrapper, AFTER our hook has already finalised the span), but - §4.5 dedup would still promote them to canonical → zero-token + backend dedup would still promote them to canonical → zero-token LLMUsageEvent. Leaving the parent ``bedrock.completion`` / ``bedrock.converse`` span (which DOES get full usage from ``stream_done``) as the canonical event. Streaming retry-loop - detection is the deferred follow-up - ``ST-10.4-FOLLOWUP-streaming-usage``. + detection is a deferred streaming-usage follow-up. """ try: if request is None: @@ -367,7 +364,7 @@ def _before_send_hook(event_name: Optional[str] = None, request: Any = None, **_ _set_parent_marker(parent) ctx[_CTX_SPAN_KEY] = span except Exception: - logger.debug("ST-10.4: bedrock before-send hook failed", exc_info=True) + logger.debug("Bedrock retry emitter: before-send hook failed", exc_info=True) class _ResponseDictAdapter: @@ -413,8 +410,7 @@ def _response_received_hook( * ``http_response`` + ``parsed`` (older botocore signature the fork-side unit tests drive directly). * ``response_dict`` + ``parsed_response`` (botocore 1.42.x — the - names the real event emitter actually uses; verified via - ST-10.6 fr-system-tests probe). + names the real event emitter actually uses). Per-attempt finalisation reads ``status_code`` and ``headers`` from whichever shape is present; the rest of the body just falls through @@ -449,7 +445,7 @@ def _response_received_hook( except Exception: pass except Exception: - logger.debug("ST-10.4: bedrock response-received hook failed", exc_info=True) + logger.debug("Bedrock retry emitter: response-received hook failed", exc_info=True) def install_event_hooks_on_client(client: Any, tracer_provider=None) -> None: @@ -476,7 +472,7 @@ def install_event_hooks_on_client(client: Any, tracer_provider=None) -> None: events = getattr(events, "events", None) if events is not None else None if events is None: logger.debug( - "ST-10.4: bedrock client missing .meta.events; " + "Bedrock retry emitter: client missing .meta.events; " "skipping retry_attempt event-hook registration" ) return @@ -492,7 +488,7 @@ def install_event_hooks_on_client(client: Any, tracer_provider=None) -> None: ) except Exception: logger.warning( - "ST-10.4: failed to register bedrock retry_attempt event hooks; " + "Bedrock retry emitter: failed to register retry_attempt event hooks; " "retry_attempt emission disabled for this client", exc_info=True, ) @@ -529,7 +525,7 @@ def uninstall_event_hooks_on_client(client: Any) -> None: except Exception: pass except Exception: - logger.debug("ST-10.4: bedrock event-hook unregister failed", exc_info=True) + logger.debug("Bedrock retry emitter: event-hook unregister failed", exc_info=True) def _reset_tracer_provider_for_test() -> None: diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/fortifyroot/retry_registry.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/fortifyroot/retry_registry.py index a548991..3c11cd9 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/fortifyroot/retry_registry.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/fortifyroot/retry_registry.py @@ -1,4 +1,4 @@ -"""§4.7.1 token-based framework-attempt registry. +"""Token-based framework-attempt registry. Shared across all FR fork instrumentations. The flow is: @@ -23,14 +23,13 @@ the duration of one attempt. (Re-entrancy across threads is handled by the per-TID dict.) -Design references: - - RETRY_LOOP.md §4.7 — suppression discipline rationale. - - RETRY_LOOP.md §4.7.1 — registry shape, eviction policy, required - tests (re-entrancy, stale-cleanup, - parent-end cleanup, cap-eviction, - thread-ID reuse). - - phase_st10_retryloop.txt round-3 disposition — replaces an - earlier ``set[int]`` design that didn't handle re-entrancy. +Design notes: + - The registry owns suppression discipline for framework retry loops. + - The shape, eviction policy, and tests cover re-entrancy, + stale-cleanup, parent-end cleanup, cap-eviction, and thread-ID + reuse. + - The token map replaces an earlier per-thread set design that did + not handle re-entrant attempts. """ from __future__ import annotations @@ -44,7 +43,7 @@ logger = logging.getLogger(__name__) -# Module-level state (per process). Per RETRY_LOOP.md §4.7.1: +# Module-level state (per process): # shape: dict[tid, dict[token, started_at_monotonic_seconds]] # Keying by tid (thread ID) is what makes per-thread ownership work; # keying by token within a TID is what makes re-entrancy work (one @@ -227,8 +226,7 @@ def is_framework_owned(tid: Optional[int] = None) -> bool: truth for that logical call. Performs per-TID stale eviction in-band before answering, so a - leaked token cannot indefinitely suppress emission (see - review-round-4 Q2 fix in phase_st10_retryloop.txt). + leaked token cannot indefinitely suppress emission. """ if tid is None: tid = threading.get_ident() diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/__init__.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/__init__.py index 3b03f59..b6d16b1 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/__init__.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/__init__.py @@ -102,8 +102,8 @@ def _instrument(self, **kwargs): traceloopCallbackHandler = TraceloopCallbackHandler( tracer, duration_histogram, token_histogram ) - # ST-10.2: register the FR retry-attempt handler alongside the - # existing Traceloop handler. Per ST-10.0 C2 POC findings, this + # Register the FR retry-attempt handler alongside the + # existing Traceloop handler. This # captures per-HTTP-attempt callbacks on framework-layer retry # paths (e.g. Runnable.with_retry) and emits one # fortifyroot.langchain.attempt_ sibling span per attempt @@ -111,7 +111,7 @@ def _instrument(self, **kwargs): # handler as a CONSTRUCTION-time reference so it can resolve the # workflow parent via Traceloop's run_id-keyed ``spans`` dict # without mutating shared state per-callback-manager-init - # (review-round-2 Major 5). + # shared-state races. fortifyrootRetryHandler = _FortifyRootRetryHandler( traceloop_handler=traceloopCallbackHandler, ) @@ -253,10 +253,8 @@ def __call__( # the test-isolation bug observed when the FR retry handler # was registered FIRST: running before Traceloop caused # Traceloop's context-attach/detach discipline to break - # downstream (LiteLLM tests in the same pytest session - # observed stale OTel ambient context leaking from LangChain - # — review-batch-1 v6 trace-id-shared-across-tests bug, - # 2026-05-11). + # downstream tests from observing stale OTel ambient context + # leaking from LangChain. for handler in instance.inheritable_handlers: if isinstance(handler, type(self._callback_handler)): break @@ -266,14 +264,14 @@ def __call__( # we need a way to determine the type of CallbackManager being wrapped. self._callback_handler._callback_manager = instance instance.add_handler(self._callback_handler, True) - # ST-10.2: register the FR retry-attempt handler AFTER + # Register the FR retry-attempt handler AFTER # Traceloop. Idempotent registration. The handler already # holds a CONSTRUCTION-time reference to the Traceloop handler # (set in LangchainInstrumentor._instrument). We deliberately # do NOT mutate any shared attribute on the retry handler per # callback-manager-init — that previously raced when concurrent # Runnable invocations created BaseCallbackManagers on - # different threads (review-round-2 Major 5). + # different threads. if self._retry_handler is not None: for handler in instance.inheritable_handlers: if isinstance(handler, type(self._retry_handler)): @@ -319,13 +317,13 @@ def __call__( # In legacy chains like LLMChain, suppressing model instrumentations # within create_llm_span doesn't work, so this should helps as a fallback. # - # ST-10 review-round-2 fix (2026-05-11): capture the attach token + # Capture the attach token # and detach in ``finally`` so this suppression layer doesn't leak # into the OTel context stack indefinitely. The pre-fix code # never detached, accumulating SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY # frames on every wrapped openai call, which compounded across # LangChain tests in long pytest sessions and risked corrupting - # later instrumentor behaviour. See review-round-2 Blocker 3. + # later instrumentor behaviour. suppression_token: Optional[Any] = None try: suppression_token = context_api.attach( diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/callback_handler.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/callback_handler.py index 4739a7b..69b086b 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/callback_handler.py @@ -195,7 +195,7 @@ def _end_span(self, span: Span, run_id: UUID) -> None: if child_span.end_time is None: # avoid warning on ended spans child_span.end() span.end() - # ST-10 review-round-2 fix (2026-05-11): detach ALL attached + # Detach ALL attached # tokens in LIFO order (reverse of attach). Pre-fix, only the # single ``token`` field was detached — which for LLM spans # was the suppression token only, leaving the span-context @@ -273,7 +273,7 @@ def _create_span( entity_path: str = "", metadata: Optional[dict[str, Any]] = None, ) -> Span: - # ST-10 review-round-2 fix (2026-05-11): capture every + # Capture every # context_api.attach()'s return token and append to the # SpanHolder's ``tokens`` list. ``_end_span`` detaches them in # LIFO order. Pre-fix, the metadata-association attach below @@ -284,7 +284,7 @@ def _create_span( # span-context attach, leaving the ended span "current" in OTel # context past the end of the LangChain test, which polluted # later LiteLLM / LlamaIndex tests in the same pytest session - # (session-scoped sdk_helper). See review-round-2 Blocker 1. + # session. attached_tokens: list[Any] = [] if metadata is not None: current_association_properties = ( @@ -404,7 +404,7 @@ def _create_llm_span( # we already have an LLM span by this point, # so skip any downstream instrumentation from here # - # ST-10 review-round-2 fix (2026-05-11): APPEND the suppression + # Append the suppression # token to the existing SpanHolder.tokens list rather than # replacing the SpanHolder. The pre-fix code created a new # SpanHolder with ONLY the suppression token, dropping the @@ -513,7 +513,7 @@ def on_chain_end( self._end_span(span, run_id) if parent_run_id is None: - # ST-10 review-round-2 note (2026-05-11): pre-existing leak — + # Pre-existing leak: # this attach is not paired with a detach. It writes # SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY=False to a new # context layer that grows the OTel context stack by one diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/retry_handler.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/retry_handler.py index ca4fda2..b0f4adc 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/retry_handler.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/retry_handler.py @@ -1,23 +1,21 @@ -"""ST-10.2 retry-aware emission for LangChain. +"""Retry-aware emission for LangChain. -Per RETRY_LOOP.md §4.4 LangChain row + §4.4.2 coverage limitation + -ST-10.0 C2 empirical verification (POC results in -phase_st10_retryloop.txt): +Design contract: - - Hook BOTH ``on_chat_model_start`` (chat models, F1 finding) + - Hook BOTH ``on_chat_model_start`` (chat models) AND ``on_llm_start`` (legacy completion LLMs). - Per-HTTP-attempt firing is verified on framework-layer retry paths (e.g. ``Runnable.with_retry``); provider-SDK-internal retries (e.g. ``ChatOpenAI(max_retries=N)``) fire callbacks - once per logical call → §4.4.2 coverage limitation applies. + once per logical call, so the framework coverage limitation applies. - Use ``run_id`` as the per-attempt correlation key. - Emit ``fortifyroot.langchain.attempt_`` sibling spans under the parent_run_id's span (NOT under the per-LLM Traceloop span — siblings need to share a parent for RetryDetectorProc's grouping to work). - - §4.7.1: register/unregister framework-attempt tokens so - direct-SDK wrappers (ST-10.4) suppress their own emission. - - §4.5: set has_attempt_child=true on the parent AFTER + - Register/unregister framework-attempt tokens so direct-SDK + wrappers suppress their own emission. + - Set has_attempt_child=true on the parent AFTER the first qualifying llm_attempt successfully starts. - Attempt numbering is conservative for LangChain: callbacks expose a workflow parent that can contain multiple unrelated LLM calls, so @@ -28,7 +26,7 @@ TraceloopCallbackHandler (not as a replacement) — it captures metadata only (model, tokens, status), so its placement is not safety-critical. The existing handler continues to do prompt / -completion attribute capture; the §4.5 backend dedup correctly +completion attribute capture; backend dedup correctly skips its per-attempt spans because they're non-retry siblings of the llm_attempts emitted here. """ @@ -64,18 +62,18 @@ logger = logging.getLogger(__name__) -# ST-10 §4.4: per-attempt sibling span name + role. +# Per-attempt sibling span name + role. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.langchain" _FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" -# ST-10 §4.5 parent marker. +# Parent marker. _FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY # ---------------------------------------------------------------------- # Per-attempt correlation map. Key = LangChain run_id (UUID per attempt -# on framework-layer retry paths — verified ST-10.0 C2 POC). Value = +# on framework-layer retry paths). Value = # {span, started_at_monotonic, framework_token, ended}. Bounded-size + # TTL eviction defends against framework crashes that leave attempts # open. Mirrors the LiteLLM map's shape for consistency across wrappers. @@ -194,7 +192,7 @@ def _resolve_parent_span( retry_attempts must share one OTel parent → RetryDetectorProc can group them. - Resolution strategy (revised 2026-05-11 after review-round-2): + Resolution strategy: Strategy A — TRACELOOP SPANS-DICT LOOKUP (load-bearing when parent_run_id is set): @@ -209,7 +207,7 @@ def _resolve_parent_span( The ``traceloop_handler`` is passed as a DIRECT reference at ``_FortifyRootRetryHandler`` construction time (NOT via a mutable shared back-reference to BaseCallbackManager). This - fixes the review-round-2 Major-5 concern: a single shared + avoids a single shared ``_FortifyRootRetryHandler`` instance was being mutated per-BaseCallbackManager-init, racing concurrent callbacks onto the wrong manager's spans dict. @@ -226,14 +224,14 @@ def _resolve_parent_span( its per-LLM span as the OTel ambient, so ambient is the correct workflow parent. - No-emission policy (review-round-2 Blocker 2): + No-emission policy: If parent_run_id IS set AND traceloop_handler is provided (production wiring) BUT Strategy A's Traceloop lookup fails (parent_run_id not in its spans dict — e.g. evicted, stale), DO NOT fall back to ambient. Ambient at that moment is likely Traceloop's per-LLM span (handler-order is Traceloop-first) and parenting under it would break sibling-grouping. Return - None; the caller skips emission with a debug log. The §4.5 + None; the caller skips emission with a debug log. Backend backend dedup degrades gracefully (no retry_attempt → parent stays canonical → single LLMUsageEvent per call). """ @@ -327,7 +325,7 @@ def _add_prompt_attrs( ) -> None: """Copy LangChain's request content onto the retry_attempt span. - Backend §4.5 makes retry_attempt the canonical LLMUsageEvent span + Backend dedup makes retry_attempt the canonical LLMUsageEvent span when it exists. Safety E2E tests and customers looking up the canonical event therefore still need the same prompt content that Traceloop's normal LLM span carries. The callback receives prompts @@ -381,7 +379,7 @@ def _start_retry_attempt( parent_span = _resolve_parent_span(parent_run_id, traceloop_handler=traceloop_handler) if parent_span is None: # No ambient parent → orphan retry_attempt would have no place - # in the trace tree. Skip emission. The §4.5 backend dedup + # in the trace tree. Skip emission. Backend dedup # degrades gracefully when no retry_attempt exists. logger.debug( "no ambient parent span for langchain retry_attempt; skipping emission " @@ -447,7 +445,7 @@ def _start_retry_attempt( "ended": False, } - # §4.5 marker timing: set AFTER the first qualifying llm_attempt + # Parent-marker timing: set AFTER the first qualifying llm_attempt # has successfully started under this parent. Idempotent — setting # the attribute twice on the same parent is a no-op. try: @@ -537,7 +535,7 @@ class _FortifyRootRetryHandler(BaseCallbackHandler): """LangChain BaseCallbackHandler that emits one fortifyroot.langchain.attempt_ sibling span per LLM-start callback invocation. Hooks BOTH on_chat_model_start (chat models — - F1 finding from ST-10.0 C2 POC) AND on_llm_start (legacy + observed chat-model path) AND on_llm_start (legacy completion LLMs). Key correlation: ``run_id`` (UUID minted per attempt by LangChain). @@ -560,8 +558,7 @@ class _FortifyRootRetryHandler(BaseCallbackHandler): run_inline: bool = True # Set by ``_BaseCallbackManagerInitWrapper`` at construction time - # (NOT mutated per-callback-manager-init — see review-round-2 - # Major-5). Direct reference to the sibling Traceloop handler whose + # (NOT mutated per-callback-manager-init). Direct reference to the sibling Traceloop handler whose # ``spans`` dict we look up by run_id to resolve the workflow # parent for sibling-grouping across multi-attempt retries. See # ``_resolve_parent_span``. diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/span_utils.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/span_utils.py index efb23a0..718b5e5 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/span_utils.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/langchain/span_utils.py @@ -41,7 +41,7 @@ class SpanHolder: start_time: float = field(default_factory=time.perf_counter) request_model: Optional[str] = None streaming_first_token_time: Optional[float] = None - # ST-10 review-round-2 fix (2026-05-11): every context_api.attach() + # Every context_api.attach() # performed for this span — span-context, suppression, metadata # association_properties — appended here in attach order. ``_end_span`` # detaches them in REVERSE order (LIFO) so OTel's context stack is diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/litellm/__init__.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/litellm/__init__.py index 22906d1..e9cb4d7 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/litellm/__init__.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/litellm/__init__.py @@ -61,12 +61,12 @@ # ``buildSafetyWrapperDedupeSet`` cannot observe siblings across batches. _FR_HAS_NATIVE_OTEL_CHILD_KEY = "fortifyroot.span.has_native_otel_child" -# ST-10 §4.5: marker on parent set AFTER the first qualifying llm_attempt +# Marker on parent set AFTER the first qualifying llm_attempt # child has started, so the FR backend's LLMUsageExtractor can dedup the # parent + non-attempt siblings cross-batch (mirrors has_native_otel_child). _FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY -# ST-10 §4.4: per-attempt sibling span emitted by _FortifyRootRetryEmitter +# Per-attempt sibling span emitted by _FortifyRootRetryEmitter # under the safety_wrapper parent. _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.litellm" _FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" @@ -187,8 +187,7 @@ async def _ensure_completion_safety_applied_async( # isn't installed (the instrumentor's ``instrumentation_dependencies`` check # guards actual use). # -# Discovered end-to-end during ST-10 review-batch-1 re-verification 2026-05-10 -# after local vendoring. The pre-existing _FortifyRootCompletionLogger is +# Discovered end-to-end after local vendoring. The pre-existing _FortifyRootCompletionLogger is # also duck-typed (same latent bug) but its primary safety-masking path is # synchronous inside _finalize_response, so its callback never firing is # masked in production. The retry emitter has no such backup — purely @@ -200,22 +199,22 @@ async def _ensure_completion_safety_applied_async( # ---------------------------------------------------------------------- -# ST-10 §4.4 / §4.3: retry-attempt sibling-span emission via a second +# Retry-attempt sibling-span emission via a second # LiteLLM CustomLogger. # ---------------------------------------------------------------------- # # The emitter opens one ``fortifyroot.litellm.attempt_`` sibling span # per attempt-start callback fired by LiteLLM, and ends it on the matching -# success/failure callback. Per ST-10.0 C1 source-verified findings, this +# success/failure callback. This # fires per-attempt only on the ``completion_with_retries(num_retries=N)`` -# and ``Router(...)`` retry surfaces — see RETRY_LOOP.md §4.4.2 for the -# documented coverage limitation on the ``completion(num_retries=N)`` -# path (where retries delegate to the underlying provider SDK and are +# and ``Router(...)`` retry surfaces. There is a documented coverage +# limitation on the ``completion(num_retries=N)`` path (where retries +# delegate to the underlying provider SDK and are # invisible to LiteLLM's callback layer). -# Per-attempt correlation map (§4.3). Key = LiteLLM's per-call +# Per-attempt correlation map. Key = LiteLLM's per-call # ``litellm_call_id`` (sufficient because each attempt at the -# observable surfaces gets a fresh ID — verified ST-10.0 C2 POC). Value = +# observable surfaces gets a fresh ID). Value = # {span, started_at_monotonic, parent_span, framework_token, ended}. # Bounded-size + TTL eviction defends against framework crashes that # leave attempts open. @@ -327,7 +326,7 @@ def _set_active_retry_attempt_attribute(parent, key: str, value) -> None: def _resolve_routed_provider(kwargs) -> Optional[str]: """Best-effort: derive the ROUTED provider (e.g. ``openai``) from LiteLLM kwargs for the retry_attempt span's ``gen_ai.system`` - attribute. Per RETRY_LOOP.md §4.2, this is the routed provider + attribute. This is the routed provider (NOT the framework name). Falls back to ``litellm`` if undetermined. """ candidate = ( @@ -353,7 +352,7 @@ def _resolve_routed_provider(kwargs) -> Optional[str]: # ``claude-4-sonnet-20250514``) even when the public # call used ``anthropic/``. Without this inference # retry_attempt spans fall back to gen_ai.system="litellm", - # so the backend stores the canonical ST-10 event under the + # so the backend stores the canonical event under the # framework rather than the routed provider. raw = "anthropic" elif model and "." in model: @@ -374,13 +373,13 @@ def _resolve_routed_provider(kwargs) -> Optional[str]: # Normalisation for the ``gen_ai.system`` attribute. # -# Per RETRY_LOOP.md §4.2, the value MUST be the ROUTED provider, NOT +# The value MUST be the ROUTED provider, NOT # the framework, AND it MUST be the canonical OTel-semconv form (e.g. # Bedrock = ``"AWS"``). LiteLLM's ``custom_llm_provider`` field uses # its own taxonomy (``"bedrock"``, ``"bedrock_converse"``, -# ``"sagemaker"``, ...), so we map those to the §4.2 canonical -# values and leave already-canonical values untouched. (Review-batch-1 -# Minor 4 fix 2026-05-10 — keeps cross-wrapper consistency with +# ``"sagemaker"``, ...), so we map those to canonical +# values and leave already-canonical values untouched. This keeps +# cross-wrapper consistency with # LangChain's _resolve_routed_provider which already normalises # langchain_aws → ``"AWS"``.) _LITELLM_PROVIDER_NORMALISATION = { @@ -399,7 +398,7 @@ def _resolve_routed_provider(kwargs) -> Optional[str]: def _normalize_routed_provider(raw: str) -> str: - """Map LiteLLM's provider taxonomy to RETRY_LOOP.md §4.2's + """Map LiteLLM's provider taxonomy to the retry-attempt contract's routed-provider form. If no mapping applies, return the raw value lower-cased (matches OpenAI / Anthropic which already use the canonical form). @@ -452,7 +451,7 @@ def _add_retry_attempt_prompt_attrs( ) -> None: """Copy request prompt content onto the retry_attempt span. - Backend §4.5 makes retry_attempt the canonical LLMUsageEvent span + Backend dedup makes retry_attempt the canonical LLMUsageEvent span when it exists. Safety correlation and masking assertions therefore need the same request content on the retry_attempt span that the safety_wrapper parent carries. @@ -514,7 +513,7 @@ def _start_retry_attempt_span(kwargs, *, is_text_completion: bool = False) -> No attrs[GenAIAttributes.GEN_AI_REQUEST_MODEL] = str(model) server = _server_address(kwargs) if server: - # Per RETRY_LOOP.md §4.2, attribute name is the OTel-standard + # Attribute name is the OTel-standard # ``server.address`` (network namespace) — using the literal # key string for forward-compat across semconv lib changes. attrs["server.address"] = server @@ -533,7 +532,7 @@ def _start_retry_attempt_span(kwargs, *, is_text_completion: bool = False) -> No context=parent_ctx, ) - # §4.7.1: register a framework-attempt token so direct-SDK + # Register a framework-attempt token so direct-SDK # wrappers (OpenAI/Anthropic/Bedrock) suppress their own emission # while this attempt is in flight. try: @@ -557,7 +556,7 @@ def _start_retry_attempt_span(kwargs, *, is_text_completion: bool = False) -> No "ended": False, } - # §4.5 marker timing: set has_attempt_child=true on the + # Parent-marker timing: set has_attempt_child=true on the # parent ONLY AFTER the first qualifying llm_attempt has # successfully started. We just succeeded; mark the parent now. # Idempotent: setting the attribute twice on the same parent is a @@ -615,7 +614,7 @@ def _finalize_retry_attempt_span( output_tokens = get_object_value(usage, "completion_tokens") if output_tokens is None: output_tokens = get_object_value(usage, "output_tokens") - # Per §4.2 token-usage rule: SET when known, OMIT when unknown. + # Token-usage rule: SET when known, OMIT when unknown. if input_tokens is not None: span.set_attribute(GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS, int(input_tokens)) if output_tokens is not None: @@ -625,7 +624,7 @@ def _finalize_retry_attempt_span( exception = kwargs.get("exception") if isinstance(kwargs, dict) else None status_code = _httpx_status_code(exception) or _httpx_status_code(response_obj) if status_code is not None: - # Per RETRY_LOOP.md §4.2 the attribute name is the + # The attribute name is the # OTel-standard ``http.status_code`` (legacy semconv; # backend extractor reads the literal key, not a # python-binding constant). @@ -778,7 +777,7 @@ def _instrument(self, **kwargs): litellm.callbacks = [] self._fr_logger = _FortifyRootCompletionLogger() litellm.callbacks.insert(0, self._fr_logger) - # ST-10.1: register the retry-attempt emitter immediately + # Register the retry-attempt emitter immediately # AFTER the completion logger so completion-safety masking # still runs first. The retry emitter captures metadata # only (model, tokens, status), so its placement is not @@ -873,7 +872,7 @@ def _invoke_completion(tracer, wrapped, args, kwargs, *, is_text_completion=Fals if _native_otel_callback_active(): # Hint to the FR backend that this safety_wrapper WILL have a # sibling litellm_request child emitted by LiteLLM's native OTel - # callback — enables single-pass dedup in proc_llm_extractor.go + # callback — enables single-pass dedup in backend LLM extractor # without needing to see the child in the same OTLP batch. span_attrs[_FR_HAS_NATIVE_OTEL_CHILD_KEY] = True span = tracer.start_span( diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/dispatcher_wrapper.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/dispatcher_wrapper.py index e025911..8669e42 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/dispatcher_wrapper.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/dispatcher_wrapper.py @@ -156,7 +156,7 @@ def _stamp_llm_response_model_for_safety(event: BaseEvent, span) -> None: def instrument_with_dispatcher(tracer: Tracer): instrument_llm_safety_wrappers() dispatcher = get_dispatcher() - # ST-10.3: register the FR retry-attempt handler FIRST, before + # Register the FR retry-attempt handler FIRST, before # OpenLLMetrySpanHandler. Order matters because span handlers # fire in registration order — and our handler reads the # ambient OTel context to decide the parent of the @@ -383,7 +383,7 @@ def new_span( # ``_stamp_llm_model_for_safety()`` to keep safety findings # emitted on this span (via emit_deferred_findings) correctly # attributed to model / provider. See - # fr-system-tests st_phase_6.txt addendum 8. + # framework safety attribution regression coverage. if is_openllmetry_class: span.set_attribute(_FR_LLM_WRAPPER_ROLE_KEY, _FR_LLM_WRAPPER_ROLE_VALUE) try: diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/retry_handler.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/retry_handler.py index 4bdf609..d805fc8 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/retry_handler.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/retry_handler.py @@ -1,21 +1,19 @@ -"""ST-10.3 retry-aware emission for LlamaIndex. +"""Retry-aware emission for LlamaIndex. -Per RETRY_LOOP.md §4.4 LlamaIndex row + §4.4.2 coverage limitation + -ST-10.0 C3 empirical verification (POC findings F4-F6 in -phase_st10_retryloop.txt): +Design contract: - LlamaIndex's ``OpenAI.chat()`` (and equivalents) fires the dispatcher span TWICE per HTTP attempt — once on the public ``chat()`` method and once on the inner ``_chat()`` (F4). A naive event-handler approach emits 2x retry_attempt children - per attempt. ST-10.3 hooks the SPAN HANDLER (not the event + per attempt. This hooks the SPAN HANDLER (not the event handler) and filters to OUTER public methods only via a method-name whitelist + ``BaseLLM`` instance check. - Per-attempt firing is verified empirically on framework-layer retry paths (e.g. tenacity wrapping at the application layer). Provider-SDK-internal retries (e.g. ``OpenAI(max_retries=N).chat(...)``) fire dispatcher spans ONCE - per logical call → §4.4.2 coverage limitation applies (same + per logical call, so the framework coverage limitation applies (same pattern as LiteLLM C1 / LangChain C2). - ``span_enter``/``span_exit``/``span_drop`` lifecycle hooks correlate cleanly: each enter has a matching exit OR drop, so @@ -58,12 +56,12 @@ logger = logging.getLogger(__name__) -# ST-10 §4.4: per-attempt sibling span name + role. +# Per-attempt sibling span name + role. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.llamaindex" _FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" -# ST-10 §4.5 parent marker. +# Parent marker. _FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY # Dispatcher span IDs follow the pattern "ClassName.method-uuid". @@ -217,7 +215,7 @@ def _content_to_string(content: Any) -> str: def _add_prompt_attrs(attrs: dict[str, Any], bound_args: Any) -> None: """Copy request prompt content onto the retry_attempt span. - Backend §4.5 makes retry_attempt the canonical LLMUsageEvent span + Backend dedup makes retry_attempt the canonical LLMUsageEvent span when it exists, so safety correlation still needs prompt content on this span. LlamaIndex safety wrappers have already processed the bound arguments by the time dispatcher span handlers see them. @@ -326,7 +324,7 @@ def _start_retry_attempt(id_: str, instance: Any, bound_args: Any = None) -> Non "ended": False, } - # §4.5 marker timing: set on parent AFTER the first qualifying + # Parent-marker timing: set on parent AFTER the first qualifying # llm_attempt has successfully started under it. try: parent_span.set_attribute(_FR_HAS_ATTEMPT_CHILD_KEY, True) @@ -407,8 +405,7 @@ class _FortifyRootRetryHandler(BaseSpanHandler): public LLM method invocation (chat/achat/complete/acomplete/...). De-dup vs the inner ``_chat``/``_complete`` spans is enforced by - a method-name whitelist in ``_is_outer_llm_method`` (F4 finding - from ST-10.0 C3 POC). So one HTTP attempt = exactly one + a method-name whitelist in ``_is_outer_llm_method``. So one HTTP attempt = exactly one retry_attempt span — even though LlamaIndex internally fires dispatcher spans on both the outer and inner methods. diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/safety.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/safety.py index 7a278ef..a3e7ae7 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/safety.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/safety.py @@ -51,8 +51,8 @@ def _infer_llm_provider(model) -> str | None: model_name = str(model or "").lower() - # TODO(ST-6 follow-up): MVP support here is intentionally limited to the - # providers exercised in ST-6 (OpenAI + Anthropic). Expand this inference + # TODO: support here is intentionally limited to the currently + # certified providers (OpenAI + Anthropic). Expand this inference # when we certify more LlamaIndex-backed providers so safety-emitted # wrapper spans continue to carry provider/model attribution for them. if "claude" in model_name or "anthropic" in model_name: diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/span_utils.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/span_utils.py index bc401db..c429d62 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/span_utils.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/llamaindex/span_utils.py @@ -18,8 +18,8 @@ def _infer_llm_provider(model) -> str | None: model_name = str(model or "").lower() - # TODO(ST-6 follow-up): MVP support here is intentionally limited to the - # providers exercised in ST-6 (OpenAI + Anthropic). Expand this inference + # TODO: support here is intentionally limited to the currently + # certified providers (OpenAI + Anthropic). Expand this inference # as additional LlamaIndex-backed providers are certified so delegated # wrapper spans and backend LLMUsage attribution keep working for them too. if "claude" in model_name or "anthropic" in model_name: @@ -114,7 +114,7 @@ def set_llm_chat_response_model_attributes(event, span): output_tokens = None total_tokens = None - # TODO(ST-6 follow-up): token extraction below currently covers the usage + # TODO: token extraction below currently covers the usage # shapes we needed for the MVP providers/certified paths (OpenAI-style, # Anthropic-style, and Cohere metadata fallback). Extend this branch as # additional LlamaIndex-backed providers are added so token attribution diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/retry_handler.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/retry_handler.py index 72dd3bd..07b9407 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/retry_handler.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/retry_handler.py @@ -1,7 +1,6 @@ -"""ST-10.4 retry-aware emission for the OpenAI direct SDK. +"""Retry-aware emission for the OpenAI direct SDK. -Per RETRY_LOOP.md §4.4 OpenAI row + §4.7 suppression discipline + -ST-10.0 hook-table addendum (in phase_st10_retryloop.txt): +Design contract: - Hook the SDK-internal private httpx wrapper classes ``openai._base_client.SyncHttpxClientWrapper.send`` and @@ -24,9 +23,8 @@ (``/v1/chat/completions``, ``/v1/completions``, ``/v1/embeddings``, ``/v1/responses``, plus Azure ``/openai/deployments/.../chat/completions``). Non-LLM SDK traffic (e.g. ``/v1/models`` for token-refresh / model - listing) does NOT emit retry_attempt — per §4.4.1 allow-listing - requirement. - - Suppression discipline (§4.7): before emitting, check BOTH the + listing) does NOT emit retry_attempt. + - Suppression discipline: before emitting, check BOTH the OTel context ``SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY`` AND the shared ``is_framework_owned()`` registry. If either says "suppress", we skip emission. This is the key invariant that @@ -34,8 +32,8 @@ DOUBLE-emitting when both framework wrappers and direct-SDK wrappers are active. - Parent resolution: use the current OTel ambient span. If invalid - or absent, gracefully skip (no orphan retry_attempt). The §4.5 - backend dedup degrades gracefully. + or absent, gracefully skip (no orphan retry_attempt). Backend + dedup degrades gracefully. - Per-attempt span lifetime: open on wrap entry, end on wrap exit (success OR exception). On non-2xx HTTP, set ``http.status_code`` + ``error.type`` and ERROR status; on 2xx, @@ -71,7 +69,7 @@ logger = logging.getLogger(__name__) -# ST-10 §4.4 / §4.5 constants. +# Retry-attempt constants. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.openai" _FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" @@ -227,7 +225,7 @@ def _server_attrs_from_request(request: Any) -> dict[str, Any]: def _is_suppressed() -> bool: - """§4.7: skip emission if EITHER suppression signal is active. + """Skip emission if EITHER suppression signal is active. Subtle: the OpenAI ``chat_wrapper`` (and ``achat_wrapper``) sets ``SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY`` around its own @@ -270,7 +268,7 @@ def _resolve_parent_span() -> Optional["trace.Span"]: def _set_parent_marker(parent_span: "trace.Span") -> None: - """§4.5: mark the parent as 'has llm_attempt child' AFTER the + """Mark the parent as 'has llm_attempt child' AFTER the first child has successfully started. Idempotent — setting twice is a no-op.""" try: @@ -317,15 +315,15 @@ def _finalize_success(span: "trace.Span", response: Any, *, is_streaming: bool = For non-streaming responses (regardless of 2xx vs non-2xx), parse the JSON body and extract usage tokens + response id + response - model. Per RETRY_LOOP.md §4.4 token-usage rule (around line 164): + model. Per the retry-attempt token-usage rule: wrappers MUST extract usage from the response body whenever it's present, regardless of whether the attempt succeeded — some failures (context-length-exceeded errors etc.) DO consume tokens - and the provider returns usage in the error body. Backend §4.5 + and the provider returns usage in the error body. Backend dedup makes a qualifying retry_attempt canonical (even single- attempt) and reads token attrs from it - (``fr-backend/internal/processing/proc_llm_extractor.go`` - ``isRetryAttempt`` block) — so usage attribution must live here. + retry-attempt dedup reads token attrs from retry_attempt spans, so + usage attribution must live here. For streaming responses (``stream=True`` passed to ``send()``), body reading would consume the SSE stream before the SDK can @@ -351,7 +349,7 @@ def _finalize_success(span: "trace.Span", response: Any, *, is_streaming: bool = pass # Non-streaming body parse — applies to BOTH 2xx and non-2xx per the - # §4.4 token-usage rule. ``_extract_usage_from_body`` is fully + # retry-attempt token-usage rule. ``_extract_usage_from_body`` is fully # defensive: missing fields, parse failure, and bodies without # ``usage`` all degrade silently to "no attrs set". if not is_streaming: @@ -475,31 +473,31 @@ def _sync_send_wrapper(wrapped, instance, args, kwargs): request. Framework-attempt registry: this wrapper does NOT register a token. - The §4.7.1 registry's documented contract (see ``retry_registry.py`` + The framework-attempt registry's documented contract (see ``retry_registry.py`` docstring) reserves registration for FRAMEWORK wrappers (LiteLLM / LangChain / LlamaIndex). Direct-SDK wrappers only CONSULT via ``is_framework_owned()``. Self-registering would falsely suppress other concurrent direct-SDK calls on the same OS thread — a real issue under asyncio where two tasks share a thread. - Streaming skip (ST-10.4 review-driven 2026-05-17): when + Streaming skip: when ``stream=True`` is passed to send, this wrapper SKIPS retry_attempt emission entirely. Rationale: streaming retry_attempts cannot carry token usage (the body is the SSE stream and reading it would - break the SDK), but backend §4.5 dedup makes any qualifying + break the SDK), but backend dedup makes any qualifying retry_attempt the canonical LLMUsageEvent — producing a zero-token canonical event for streaming calls. By not emitting at all, the parent ``openai.chat`` span (which DOES get full usage attribution from ``ChatStream``'s stream-completion callback) remains the canonical event. Streaming retry-loop detection is the documented - deferred follow-up ``ST-10.4-FOLLOWUP-streaming-usage``; this - explicit skip is part of that deferral. + deferred streaming-usage follow-up; this explicit skip is part of + that deferral. """ request = args[0] if args else kwargs.get("request") if request is None or not _should_emit_for(request): return wrapped(*args, **kwargs) - # ST-10.4: skip retry_attempt emission for streaming calls. + # Skip retry_attempt emission for streaming calls. if bool(kwargs.get("stream", False)): return wrapped(*args, **kwargs) @@ -542,7 +540,7 @@ async def _async_send_wrapper(wrapped, instance, args, kwargs): if request is None or not _should_emit_for(request): return await wrapped(*args, **kwargs) - # ST-10.4: skip retry_attempt emission for streaming (see + # Skip retry_attempt emission for streaming (see # _sync_send_wrapper docstring for rationale). if bool(kwargs.get("stream", False)): return await wrapped(*args, **kwargs) @@ -632,7 +630,7 @@ def instrument_retry_emitter(tracer_provider=None) -> None: ) if not sync_ok: logger.warning( - "ST-10.4: openai._base_client.%s.%s missing/incompatible; " + "OpenAI retry emitter: openai._base_client.%s.%s missing/incompatible; " "skipping sync retry_attempt emission. Normal openai " "instrumentation is unaffected.", _SYNC_WRAPPER_CLASS, _WRAPPED_METHOD, @@ -646,13 +644,13 @@ def instrument_retry_emitter(tracer_provider=None) -> None: ) except Exception as e: logger.warning( - "ST-10.4: failed to wrap openai sync httpx send (%s); " + "OpenAI retry emitter: failed to wrap openai sync httpx send (%s); " "retry_attempt emission disabled for sync path", e, ) if not async_ok: logger.warning( - "ST-10.4: openai._base_client.%s.%s missing/incompatible; " + "OpenAI retry emitter: openai._base_client.%s.%s missing/incompatible; " "skipping async retry_attempt emission. Normal openai " "instrumentation is unaffected.", _ASYNC_WRAPPER_CLASS, _WRAPPED_METHOD, @@ -666,7 +664,7 @@ def instrument_retry_emitter(tracer_provider=None) -> None: ) except Exception as e: logger.warning( - "ST-10.4: failed to wrap openai async httpx send (%s); " + "OpenAI retry emitter: failed to wrap openai async httpx send (%s); " "retry_attempt emission disabled for async path", e, ) @@ -687,7 +685,7 @@ def uninstrument_retry_emitter() -> None: unwrap(f"{_OPENAI_BASE_CLIENT_MODULE}.{cls}", _WRAPPED_METHOD) except Exception: logger.debug( - "ST-10.4: openai unwrap of %s.%s failed (likely " + "OpenAI retry emitter: unwrap of %s.%s failed (likely " "wrap was never installed for this variant)", cls, _WRAPPED_METHOD, exc_info=True, diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/shared/chat_wrappers.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/shared/chat_wrappers.py index f2e6877..8ff5623 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/shared/chat_wrappers.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/shared/chat_wrappers.py @@ -78,8 +78,7 @@ # FR: stable internal span attributes carrying streaming latency as integer # milliseconds, so the backend can extract TTFT/STTG into RDS llm_usage_events. -# Additive to (not a replacement for) the Mimir streaming histograms. Contract: -# fr-backend/docs/development/STREAMING_LATENCY_TTFT_STTG_PLAN.md +# Additive to (not a replacement for) the Mimir streaming histograms. FR_STREAMING_TIME_TO_FIRST_TOKEN_MS = "fortifyroot.llm.streaming.time_to_first_token_ms" FR_STREAMING_TIME_TO_GENERATE_MS = "fortifyroot.llm.streaming.time_to_generate_ms" @@ -127,7 +126,7 @@ def chat_wrapper( run_async(_handle_request(span, kwargs, instance)) try: start_time = time.perf_counter() - # ST-10.4 (review-driven 2026-05-16): set + # Set # OPENAI_DIRECT_RETRY_PARENT_ACTIVE_KEY alongside the # existing SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY so # the FortifyRoot retry handler can distinguish this @@ -248,7 +247,7 @@ async def achat_wrapper( try: start_time = time.perf_counter() - # ST-10.4 (review-driven 2026-05-16): see sync chat_wrapper + # See sync chat_wrapper # above for the rationale on OPENAI_DIRECT_RETRY_PARENT_ACTIVE_KEY. attempt_ctx = context_api.set_value( SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY, True, diff --git a/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/v1/__init__.py b/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/v1/__init__.py index 7d640a4..7b644a3 100644 --- a/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/v1/__init__.py +++ b/src/fortifyroot/_vendor/opentelemetry/instrumentation/openai/v1/__init__.py @@ -358,7 +358,7 @@ def _instrument(self, **kwargs): realtime_connect_wrapper(tracer), ) - # ST-10.4: per-attempt retry_attempt emission via private + # Per-attempt retry_attempt emission via private # ``openai._base_client`` httpx wrapper classes. Guarded against # missing private symbols (logs warning + skips emission). Pass # the same tracer_provider the rest of the instrumentor uses so @@ -370,7 +370,7 @@ def _instrument(self, **kwargs): instrument_retry_emitter(tracer_provider=tracer_provider) def _uninstrument(self, **kwargs): - uninstrument_retry_emitter() # ST-10.4 symmetry + uninstrument_retry_emitter() # retry-emitter symmetry unwrap_dotted_method("openai.resources.chat.completions", "Completions.create") unwrap_dotted_method("openai.resources.completions", "Completions.create") unwrap_dotted_method("openai.resources.embeddings", "Embeddings.create") diff --git a/src/fortifyroot/core.py b/src/fortifyroot/core.py index 0215b47..56ba0b1 100644 --- a/src/fortifyroot/core.py +++ b/src/fortifyroot/core.py @@ -794,22 +794,21 @@ def span_callback(span): # May have been overridden by FORTIFYROOT_TRACE_CONTENT env above. os.environ["TRACELOOP_TRACE_CONTENT"] = str(trace_content).lower() - # ── OTel BatchSpanProcessor schedule_delay default (ST-10 MVP stopgap) ── + # OTel BatchSpanProcessor schedule_delay default for retry-loop detection. # # OpenTelemetry's BatchSpanProcessor reads OTEL_BSP_SCHEDULE_DELAY at # construction time (default = 5000 ms). With the upstream default, # any direct-SDK LLM retry chain whose total wall-clock exceeds ~5 s # (e.g. provider sends Retry-After: 10, customer has max_retries=4 # with exponential backoff, slow-failing attempts) fragments its - # spans across multiple OTLP batches → fr-backend's - # ``proc_retry_detector.go`` is strictly per-batch and never sees ≥2 - # sibling retry_attempt events together → silent RetryLoopEvent miss. + # spans across multiple OTLP batches. The backend retry-loop detector is + # per-batch in the MVP path and never sees multiple sibling retry_attempt + # events together, causing a silent RetryLoopEvent miss. # Per-attempt LLMUsageEvent extraction is unaffected. # - # The proper backend fix (DB-lookup cross-batch aggregator in - # proc_retry_detector) is deferred post-MVP as - # ``ST-10-FOLLOWUP-cross-batch-retry-detection``. As an MVP-side - # stopgap (kapil 2026-05-19), we widen the default OTel schedule + # The proper backend fix (a cross-batch retry aggregator) is deferred as + # ``cross-batch retry detection follow-up``. As an MVP-side + # stopgap (2026-05-19), we widen the default OTel schedule # delay to 15 s so nearly all real-world direct-SDK retry chains # (OpenAI / Anthropic max_retries up to 4-5 with typical # Retry-After ≤ 10 s) buffer into one OTLP batch and produce a @@ -819,12 +818,9 @@ def span_callback(span): # size (512). Customers who need a different value can override # via the standard OTel env var, which we respect via ``setdefault``. # - # See fr-backend/docs/development/RETRY_LOOP.md §1.1 for the full - # MVP-scope summary including the documented behaviour around - # disable_batch=True (which DISABLES RetryLoopEvent detection - # entirely because every span ships as its own OTLP batch — that - # is a documented MVP limitation pending the same backend - # follow-up). + # This also defines the documented behavior around disable_batch=True: + # RetryLoopEvent detection is disabled because every span ships as its + # own OTLP batch. That MVP limitation is pending the same backend follow-up. os.environ.setdefault("OTEL_BSP_SCHEDULE_DELAY", "15000") # Prepare resource attributes with FR SDK version diff --git a/tests/conftest.py b/tests/conftest.py index 01c3197..6af9f91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,12 +16,12 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter -# ST-10.4 (2026-05-17): legacy SDK tests assert exact span names / -# counts (e.g. ``assert span.name == "anthropic.chat"``). ST-10.4 +# FortifyRoot retry-attempt (2026-05-17): legacy SDK tests assert exact span names / +# counts (e.g. ``assert span.name == "anthropic.chat"``). FortifyRoot retry-attempt # emits per-attempt ``fortifyroot.*.retry_attempt`` sibling spans # under every direct-SDK LLM call. Filter those out of the standard # ``span_exporter`` fixture so upstream assertions remain valid. -# ST-10.4-specific tests can read the raw exporter or filter by role +# Retry-attempt tests can read the raw exporter or filter by role # themselves (e.g. ``_single_span`` in ``tests/openai/test_vcr.py`` # already filters retry_attempt by ``fortifyroot.span.role``). # Mirrors the same pattern used by the fr-openllmetry-py per-package @@ -31,7 +31,7 @@ class _NoFortifyRootSpanExporter(InMemorySpanExporter): """Filter spans by ROLE (not by name prefix) so that longstanding ``fortifyroot.litellm.safety`` etc. spans which upstream tests actually want to see remain visible; only - ST-10.4-emitted retry_attempt siblings are dropped.""" + FortifyRoot retry-attempt sibling spans are dropped.""" def get_finished_spans(self): # type: ignore[override] return tuple( diff --git a/tests/openai/test_vcr.py b/tests/openai/test_vcr.py index 43d4d2e..3fff5de 100644 --- a/tests/openai/test_vcr.py +++ b/tests/openai/test_vcr.py @@ -119,14 +119,12 @@ def _openai_client(pytestconfig: pytest.Config, cassette_stem: str) -> openai.Op ) -# ST-10.4 (review-driven 2026-05-16): filter ``fortifyroot.*.retry_attempt`` -# sibling spans out of legacy single-span assertions. ST-10.4 added +# Filter FortifyRoot retry-attempt sibling spans out of legacy +# single-span assertions. Retry-attempt support added # per-attempt retry_attempt siblings under every openai/anthropic/bedrock # logical call; these tests pre-date that and assume the only span in # the exporter is the logical ``openai.chat`` span. Role-based filter -# so every provider's retry_attempt is dropped uniformly. See -# fr-system-tests/docs/development/ai-logs/st_phase_10.txt addendum -# 2026-05-16 for context. +# so every provider's retry_attempt is dropped uniformly. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" _FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" diff --git a/tests/providers/conftest.py b/tests/providers/conftest.py index 77fb20f..05d51a8 100644 --- a/tests/providers/conftest.py +++ b/tests/providers/conftest.py @@ -49,9 +49,9 @@ # Mock safety configuration # --------------------------------------------------------------------------- -# Realistic rules matching the proto SafetyRule structure from -# fr-proto/proto/config/v1/config.proto. Used to test the full safety -# pipeline: init() -> provider call (VCR) -> safety engine -> masked spans. +# Realistic rules matching the backend SDK config SafetyRule structure. Used to +# test the full safety pipeline: init() -> provider call (VCR) -> safety engine +# -> masked spans. # # Masking replacement is NOT a rule field — the engine always computes it as # [.], e.g. [PII.email-detector]. @@ -99,7 +99,7 @@ def mock_safety_rules() -> list[dict[str, Any]]: class _NoFortifyRootSpanExporter(InMemorySpanExporter): - """ST-10.4 (2026-05-17): filter ST-10.4 retry_attempt sibling + """FortifyRoot retry-attempt (2026-05-17): filter FortifyRoot retry-attempt retry_attempt sibling spans (role=retry_attempt) from upstream provider-test exporter. Role-based, NOT name-prefix-based, so legitimate ``fortifyroot.litellm.safety`` etc. spans that tests actually diff --git a/tests/test_env_mapping.py b/tests/test_env_mapping.py index e721519..96c9f0b 100644 --- a/tests/test_env_mapping.py +++ b/tests/test_env_mapping.py @@ -285,15 +285,12 @@ def test_mapping_applied_on_package_import(self): class TestOTelBSPScheduleDelayDefault: - """ST-10 MVP stopgap: ocelle.init() sets OTEL_BSP_SCHEDULE_DELAY=15000 + """ocelle.init() sets OTEL_BSP_SCHEDULE_DELAY=15000 when unset, so direct-SDK retry chains buffer into one OTLP batch and produce a RetryLoopEvent via the per-batch detector. - See ``fr-backend/docs/development/RETRY_LOOP.md`` §1.1 (MVP scope - summary) for the customer-facing behaviour this default supports, - and ``ST-10-FOLLOWUP-cross-batch-retry-detection`` in - ``fr-system-tests/SYSTEM_TESTS_PLAN.md`` for the proper backend fix - that lets us revert this default later. + This supports the documented MVP retry-loop behavior until the backend + can aggregate retry attempts across OTLP batches. """ def test_default_set_when_env_unset(self): @@ -337,8 +334,8 @@ def test_default_set_when_env_unset(self): data = json.loads(json_line) assert data["after_init"] == "15000", ( f"Expected OTEL_BSP_SCHEDULE_DELAY='15000' after init, got " - f"{data['after_init']!r}. ST-10 MVP stopgap regressed — see " - f"fortifyroot/core.py and RETRY_LOOP.md §1.1." + f"{data['after_init']!r}. The MVP retry-loop batching " + f"default regressed." ) def test_customer_override_preserved(self): diff --git a/tests/test_init.py b/tests/test_init.py index 5e87fa0..cc1e940 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -18,10 +18,9 @@ # Expected temporality mapping passed to every OTLP metrics exporter # construction — see `fortifyroot.core._cumulative_preferred_temporality` -# and the Issue-I1 banner in st_phase_8.txt for why this must be -# CUMULATIVE across the board. Computed once here so all nine existing -# `metric_exporter_cls.assert_called_once_with(...)` sites stay consistent -# if the mapping ever legitimately changes. +# for why this must be CUMULATIVE across the board. Computed once here so +# every `metric_exporter_cls.assert_called_once_with(...)` site stays +# consistent if the mapping ever legitimately changes. _EXPECTED_TEMPORALITY = _cumulative_preferred_temporality() @@ -300,7 +299,7 @@ def test_init_uses_fortifyroot_base_url_when_set(self): with mock.patch.dict( os.environ, { - "FORTIFYROOT_BASE_URL": "https://dev-api.fortifyroot.com", + "FORTIFYROOT_BASE_URL": "https://tenant.api.fortifyroot.com", "FORTIFYROOT_API_KEY": "env-key", "FORTIFYROOT_CONFIG_PROFILE_ID": "cfg-env", }, @@ -315,7 +314,7 @@ def test_init_uses_fortifyroot_base_url_when_set(self): runtime_mock.assert_called_once_with( enabled=True, - api_endpoint="https://dev-api.fortifyroot.com", + api_endpoint="https://tenant.api.fortifyroot.com", api_key="env-key", config_profile_id="cfg-env", poll_interval_seconds=60, diff --git a/tests/test_metrics_temporality.py b/tests/test_metrics_temporality.py index 9afb289..933bafd 100644 --- a/tests/test_metrics_temporality.py +++ b/tests/test_metrics_temporality.py @@ -1,4 +1,4 @@ -"""Regression tests for the OTLP metrics temporality pin (Issue ST-8 I1). +"""Regression tests for the OTLP metrics temporality pin. Context: OpenTelemetry Python's OTLPMetricExporter (both HTTP and gRPC variants) defaults ``preferred_temporality`` to ``DELTA`` for Counter, @@ -6,9 +6,8 @@ back FortifyRoot's customer-facing ``SearchMetrics`` / ``GetMetricStats`` query APIs — reject DELTA for those instrument types with HTTP 500 "invalid temporality and type combination" and drop the data point. The -result was a silent, end-to-end metric ingestion failure that went -undetected until ST-8 went to exercise ``SearchMetrics`` with SDK-emitted -data. +result was a silent, end-to-end metric ingestion failure that surfaced +when ``SearchMetrics`` was exercised with SDK-emitted data. Fix (in ``fortifyroot.core``): ``_init_default_metrics_exporter`` now builds a ``preferred_temporality`` mapping via @@ -18,11 +17,9 @@ fallback gRPC for unknown schemes). These tests lock the fix in place so a future refactor cannot silently -regress back to the library default. If a legitimate reason arises to -relax the CUMULATIVE pin, the assertion below must be updated -deliberately — with a new log entry in ``fr-system-tests/docs/development/ -ai-logs/st_phase_8.txt`` explaining why, because Prometheus-backed query -paths will break without it. +regress back to the library default. If a legitimate reason arises to relax +the CUMULATIVE pin, update the assertion deliberately and document why, +because Prometheus-backed query paths will break without it. """ from __future__ import annotations @@ -201,8 +198,7 @@ def test_live_exporter_counter_and_histogram_are_cumulative( f"{instrument_cls.__name__} temporality on live " f"{type(exporter).__name__} is {pref[instrument_cls]}, " f"expected CUMULATIVE. Prometheus/Mimir-backed query APIs " - f"will drop data points silently — see ST-8 I1 in " - f"st_phase_8.txt." + f"will drop data points silently." ) def test_live_http_exporter_with_auth_header_preserves_temporality(self): diff --git a/tests/test_safety_runtime.py b/tests/test_safety_runtime.py index f1d11dd..f27a9c1 100644 --- a/tests/test_safety_runtime.py +++ b/tests/test_safety_runtime.py @@ -503,8 +503,7 @@ def test_configure_global_safety_runtime_skips_when_endpoint_is_not_fortifyroot( [ (FORTIFYROOT_API_BASE_URL, True), ("https://api.fortifyroot.com/", True), - ("https://dev-api.fortifyroot.com", True), - ("https://staging-api.fortifyroot.com", True), + ("https://tenant.api.fortifyroot.com", True), ("http://localhost:8080", True), ("http://127.0.0.1:8080", True), ("http://[::1]:8080", True),