diff --git a/.env.example b/.env.example index fbd0c3e07..1515f29e3 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,11 @@ SKILLSPECTOR_REASONING_EFFORT= # Optional language for human-readable LLM finding text. Machine-readable values # such as rule IDs and severity values remain unchanged. SKILLSPECTOR_OUTPUT_LANGUAGE= +# Optional sampling controls. Temperature is supported by hosted providers; +# seed is forwarded only to OpenAI-compatible and Azure OpenAI endpoints. +# Unset or blank values preserve provider defaults. +SKILLSPECTOR_TEMPERATURE= # range: 0..1 +SKILLSPECTOR_SEED= # integer # For SKILLSPECTOR_PROVIDER=anthropic. ANTHROPIC_API_KEY= diff --git a/README.md b/README.md index a3af76b51..dd3cd4b80 100644 --- a/README.md +++ b/README.md @@ -586,6 +586,8 @@ Issues (2) | `OPENAI_BASE_URL` | Override the OpenAI endpoint (e.g. point at Ollama). | Optional | | `SKILLSPECTOR_REASONING_EFFORT` | Optional provider- and model-dependent reasoning-effort setting. Non-empty values are trimmed and passed through unchanged; unset or blank preserves provider-default behavior. | Optional | | `SKILLSPECTOR_OUTPUT_LANGUAGE` | Short, single-line language label (letters, numbers, spaces, `_`, or `-`; maximum 64 characters) for human-readable LLM finding text such as messages, explanations, and remediation. Rule IDs, severity values, paths, code, and other machine-readable values remain unchanged. Unset, blank, or invalid values preserve the default output language. | Optional | +| `SKILLSPECTOR_TEMPERATURE` | Optional sampling temperature from `0` to `1` for hosted providers. Unset or blank preserves the provider default. Lower values can reduce run-to-run variation but do not guarantee identical output. | Optional | +| `SKILLSPECTOR_SEED` | Optional integer sampling seed for OpenAI-compatible and Azure OpenAI providers. Other hosted providers and CLI providers do not receive it. Provider support remains model-dependent. | Optional | | `ANTHROPIC_API_KEY` | Credential for the Anthropic provider (`SKILLSPECTOR_PROVIDER=anthropic`). | Required for LLM analysis when `SKILLSPECTOR_PROVIDER=anthropic` | | `ANTHROPIC_BASE_URL` | Override the native Anthropic endpoint (default: `https://api.anthropic.com`). | Optional | | `ANTHROPIC_PROXY_ENDPOINT_URL` | Full endpoint URL for the Anthropic proxy provider (Vertex-style raw-predict). | Required when `SKILLSPECTOR_PROVIDER=anthropic_proxy` | diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 7dc072b56..577c13c46 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -38,6 +38,8 @@ make install-dev - **Provider credential**: depends on the active provider — `NVIDIA_INFERENCE_KEY` (NVIDIA), `OPENAI_API_KEY` (OpenAI), or `ANTHROPIC_API_KEY` (Anthropic). See [llm_utils.py](../src/skillspector/llm_utils.py). - **`OPENAI_BASE_URL`**: Override the OpenAI endpoint (e.g. point at Ollama). - **`SKILLSPECTOR_MODEL`**: Override default model; see [constants.py](../src/skillspector/constants.py). + - **`SKILLSPECTOR_TEMPERATURE`**: Optional hosted-provider sampling temperature from `0` to `1`. + - **`SKILLSPECTOR_SEED`**: Optional integer seed for OpenAI-compatible and Azure OpenAI providers. - **Logging**: Internal/operational logging uses the stdlib `logging` module. User-facing output (report body, errors, progress) uses Rich `console.print()`. - **Env**: `SKILLSPECTOR_LOG_LEVEL` (DEBUG, INFO, WARNING, ERROR). Default is `"WARNING"` (defined in [constants.py](../src/skillspector/constants.py)). @@ -300,6 +302,8 @@ Copy [.env.example](../.env.example) to `.env` in the project root and set value | `OPENAI_BASE_URL` | Override the OpenAI endpoint (e.g. point at Ollama). | `http://localhost:11434/v1` | | `SKILLSPECTOR_REASONING_EFFORT` | Optional provider- and model-dependent reasoning-effort setting. Non-empty values are trimmed and passed through unchanged; unset or blank preserves provider-default behavior. | `high` | | `SKILLSPECTOR_OUTPUT_LANGUAGE` | Optional short, single-line language label (letters, numbers, spaces, `_`, or `-`; maximum 64 characters) for human-readable LLM finding text. Rule IDs, severity values, paths, code, and other machine-readable values remain unchanged. Unset, blank, or invalid values preserve the default output language. | `Japanese` | +| `SKILLSPECTOR_TEMPERATURE` | Optional sampling temperature from `0` to `1` for hosted providers. Unset or blank preserves provider defaults. Lower values reduce variation but do not guarantee identical output. | `0` | +| `SKILLSPECTOR_SEED` | Optional integer sampling seed for OpenAI-compatible and Azure OpenAI providers. Provider/model support is best-effort; CLI providers ignore it. | `42` | | `ANTHROPIC_API_KEY` | Credential for `SKILLSPECTOR_PROVIDER=anthropic`. | `sk-ant-...` | | `SKILLSPECTOR_MODEL` | Override the active provider's bundled default model (see [README.md](../README.md) for per-provider defaults). For `claude_cli`, this is passed as `--model` to the `claude` binary. | `gpt-5.2` | diff --git a/src/skillspector/providers/anthropic/provider.py b/src/skillspector/providers/anthropic/provider.py index 3c56d8403..4ac4a1540 100644 --- a/src/skillspector/providers/anthropic/provider.py +++ b/src/skillspector/providers/anthropic/provider.py @@ -32,7 +32,7 @@ from pydantic import SecretStr from skillspector.providers import registry -from skillspector.providers.chat_models import resolve_reasoning_effort +from skillspector.providers.chat_models import resolve_reasoning_effort, resolve_sampling_parameters # Default endpoint; overridden by ``ANTHROPIC_BASE_URL`` when set. ANTHROPIC_BASE_URL = "https://api.anthropic.com" @@ -80,6 +80,7 @@ def create_chat_model( effort = resolve_reasoning_effort() if effort is not None: kwargs["effort"] = effort + kwargs.update(resolve_sampling_parameters()) return ChatAnthropic(**kwargs) def get_context_length(self, model: str) -> int | None: diff --git a/src/skillspector/providers/anthropic_proxy/provider.py b/src/skillspector/providers/anthropic_proxy/provider.py index 46277f577..364fcb7e0 100644 --- a/src/skillspector/providers/anthropic_proxy/provider.py +++ b/src/skillspector/providers/anthropic_proxy/provider.py @@ -53,7 +53,7 @@ from pydantic import SecretStr from skillspector.providers import registry -from skillspector.providers.chat_models import resolve_reasoning_effort +from skillspector.providers.chat_models import resolve_reasoning_effort, resolve_sampling_parameters REGISTRY_PATH = str(Path(__file__).with_name("model_registry.yaml")) @@ -244,6 +244,7 @@ def create_chat_model( effort = resolve_reasoning_effort() if effort is not None: kwargs["effort"] = effort + kwargs.update(resolve_sampling_parameters()) return _ChatAnthropicProxy(**kwargs) def get_context_length(self, model: str) -> int | None: diff --git a/src/skillspector/providers/azure_openai/provider.py b/src/skillspector/providers/azure_openai/provider.py index 253549de3..0d1653bd8 100644 --- a/src/skillspector/providers/azure_openai/provider.py +++ b/src/skillspector/providers/azure_openai/provider.py @@ -37,6 +37,7 @@ from pydantic import SecretStr from skillspector.providers import registry +from skillspector.providers.chat_models import resolve_sampling_parameters REGISTRY_PATH = str(Path(__file__).with_name("model_registry.yaml")) @@ -71,14 +72,16 @@ def create_chat_model( deployment = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "").strip() or model api_version = os.environ.get("AZURE_OPENAI_API_VERSION", "").strip() or "2024-06-01" - return AzureChatOpenAI( - azure_endpoint=endpoint, - azure_deployment=deployment, - api_key=SecretStr(api_key), - api_version=api_version, - max_tokens=max_tokens, - timeout=timeout, - ) + kwargs = { + "azure_endpoint": endpoint, + "azure_deployment": deployment, + "api_key": SecretStr(api_key), + "api_version": api_version, + "max_tokens": max_tokens, + "timeout": timeout, + } + kwargs.update(resolve_sampling_parameters(include_seed=True)) + return AzureChatOpenAI(**kwargs) def get_context_length(self, model: str) -> int | None: return registry.lookup_context_length(REGISTRY_PATH, model) diff --git a/src/skillspector/providers/bedrock/provider.py b/src/skillspector/providers/bedrock/provider.py index 2beac0d1e..1e7a6ff0e 100644 --- a/src/skillspector/providers/bedrock/provider.py +++ b/src/skillspector/providers/bedrock/provider.py @@ -41,6 +41,7 @@ from langchain_core.language_models.chat_models import BaseChatModel from skillspector.providers import registry +from skillspector.providers.chat_models import resolve_sampling_parameters BEDROCK_DEFAULT_REGION = "us-west-2" # Cross-region inference profile ID for Claude Sonnet 4.6. Public, @@ -129,6 +130,7 @@ def create_chat_model( } if model.startswith("arn:"): kwargs["provider"] = "anthropic" + kwargs.update(resolve_sampling_parameters()) return ChatBedrockConverse(**kwargs) diff --git a/src/skillspector/providers/chat_models.py b/src/skillspector/providers/chat_models.py index ec4b62af4..aa13d44ae 100644 --- a/src/skillspector/providers/chat_models.py +++ b/src/skillspector/providers/chat_models.py @@ -34,6 +34,28 @@ def resolve_reasoning_effort() -> str | None: return reasoning_effort or None +def resolve_sampling_parameters(*, include_seed: bool = False) -> dict[str, float | int]: + """Resolve optional, validated sampling controls for hosted providers.""" + parameters: dict[str, float | int] = {} + raw_temperature = os.environ.get("SKILLSPECTOR_TEMPERATURE", "").strip() + if raw_temperature: + try: + temperature = float(raw_temperature) + except ValueError as exc: + raise ValueError("SKILLSPECTOR_TEMPERATURE must be a number between 0 and 1") from exc + if not 0 <= temperature <= 1: + raise ValueError("SKILLSPECTOR_TEMPERATURE must be between 0 and 1") + parameters["temperature"] = temperature + + raw_seed = os.environ.get("SKILLSPECTOR_SEED", "").strip() + if include_seed and raw_seed: + try: + parameters["seed"] = int(raw_seed) + except ValueError as exc: + raise ValueError("SKILLSPECTOR_SEED must be an integer") from exc + return parameters + + def validate_base_url(url: str | None) -> None: """Warn if *url* is not a well-formed http(s) URL. @@ -82,4 +104,5 @@ def create_openai_compatible_chat_model( reasoning_effort = resolve_reasoning_effort() if reasoning_effort: kwargs["reasoning_effort"] = reasoning_effort + kwargs.update(resolve_sampling_parameters(include_seed=True)) return ChatOpenAI(**kwargs) diff --git a/tests/unit/test_anthropic_proxy_provider.py b/tests/unit/test_anthropic_proxy_provider.py index 72751a53f..b991cbc21 100644 --- a/tests/unit/test_anthropic_proxy_provider.py +++ b/tests/unit/test_anthropic_proxy_provider.py @@ -43,6 +43,8 @@ def _clean_env(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("SKILLSPECTOR_PROVIDER", raising=False) monkeypatch.delenv("SKILLSPECTOR_SSL_VERIFY", raising=False) monkeypatch.delenv("SKILLSPECTOR_REASONING_EFFORT", raising=False) + monkeypatch.delenv("SKILLSPECTOR_TEMPERATURE", raising=False) + monkeypatch.delenv("SKILLSPECTOR_SEED", raising=False) monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) monkeypatch.delenv("NVIDIA_INFERENCE_KEY", raising=False) @@ -134,6 +136,24 @@ def fake_proxy(**kwargs: object) -> dict[str, object]: assert "effort" not in captured + def test_temperature_is_forwarded(self, monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} + + def fake_proxy(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return kwargs + + monkeypatch.setattr( + "skillspector.providers.anthropic_proxy.provider._ChatAnthropicProxy", fake_proxy + ) + monkeypatch.setenv("ANTHROPIC_PROXY_API_KEY", "bearer-tok") + monkeypatch.setenv("ANTHROPIC_PROXY_ENDPOINT_URL", "https://proxy.example.com/predict") + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.1") + + AnthropicProxyProvider().create_chat_model("claude-sonnet-4-6", max_tokens=4096) + + assert captured["temperature"] == 0.1 + class TestAnthropicProxyProviderMetadata: """Token-budget metadata and model resolution tests.""" diff --git a/tests/unit/test_bedrock_provider.py b/tests/unit/test_bedrock_provider.py index 9c48eba28..f7a558347 100644 --- a/tests/unit/test_bedrock_provider.py +++ b/tests/unit/test_bedrock_provider.py @@ -51,6 +51,8 @@ def _clean_provider_env(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("SKILLSPECTOR_MODEL_REGISTRY", raising=False) monkeypatch.delenv("AWS_PROFILE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) + monkeypatch.delenv("SKILLSPECTOR_TEMPERATURE", raising=False) + monkeypatch.delenv("SKILLSPECTOR_SEED", raising=False) registry._load.cache_clear() yield registry._load.cache_clear() @@ -243,6 +245,28 @@ def test_plain_model_id_does_not_pin_provider( assert "provider" not in mock_chat.call_args.kwargs + @patch("skillspector.providers.bedrock.provider.ChatBedrockConverse") + @patch("skillspector.providers.bedrock.provider.boto3.Session") + def test_temperature_is_forwarded_without_openai_seed( + self, + mock_session: MagicMock, + mock_chat: MagicMock, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + mock_session.return_value.get_credentials.return_value = MagicMock() + mock_session.return_value.client.return_value = MagicMock() + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.3") + monkeypatch.setenv("SKILLSPECTOR_SEED", "42") + + BedrockProvider().create_chat_model( + "us.anthropic.claude-sonnet-4-6-20250915-v1:0", + max_tokens=1024, + ) + + kwargs = mock_chat.call_args.kwargs + assert kwargs["temperature"] == 0.3 + assert "seed" not in kwargs + class TestBedrockProviderSelection: """SKILLSPECTOR_PROVIDER=bedrock activates BedrockProvider.""" diff --git a/tests/unit/test_constants.py b/tests/unit/test_constants.py index 6cfdabc6f..1ea5125d5 100644 --- a/tests/unit/test_constants.py +++ b/tests/unit/test_constants.py @@ -37,6 +37,8 @@ def _clean_env(monkeypatch: pytest.MonkeyPatch): "OPENAI_API_KEY", "OPENAI_BASE_URL", "SKILLSPECTOR_REASONING_EFFORT", + "SKILLSPECTOR_TEMPERATURE", + "SKILLSPECTOR_SEED", "ANTHROPIC_API_KEY", ): monkeypatch.delenv(key, raising=False) diff --git a/tests/unit/test_new_providers.py b/tests/unit/test_new_providers.py index 8a86c93bf..12a3d32e0 100644 --- a/tests/unit/test_new_providers.py +++ b/tests/unit/test_new_providers.py @@ -49,6 +49,8 @@ def _clean_provider_env(monkeypatch: pytest.MonkeyPatch): "AZURE_OPENAI_API_VERSION", "SKILLSPECTOR_COMPAT_API_KEY", "SKILLSPECTOR_COMPAT_BASE_URL", + "SKILLSPECTOR_TEMPERATURE", + "SKILLSPECTOR_SEED", ): monkeypatch.delenv(key, raising=False) registry._load.cache_clear() @@ -170,6 +172,27 @@ def test_custom_api_version(self, monkeypatch: pytest.MonkeyPatch) -> None: assert isinstance(llm, AzureChatOpenAI) assert llm.openai_api_version == "2025-01-01" + def test_sampling_controls_are_forwarded(self, monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} + + def fake_azure_chat_openai(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return kwargs + + monkeypatch.setattr( + "skillspector.providers.azure_openai.provider.AzureChatOpenAI", + fake_azure_chat_openai, + ) + monkeypatch.setenv("AZURE_OPENAI_API_KEY", "azure-key") + monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "https://myorg.openai.azure.com/") + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.2") + monkeypatch.setenv("SKILLSPECTOR_SEED", "7") + + AzureOpenAIProvider().create_chat_model("gpt-4o", max_tokens=1024) + + assert captured["temperature"] == 0.2 + assert captured["seed"] == 7 + def test_default_model(self) -> None: assert AzureOpenAIProvider().resolve_model() == "gpt-4o" diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index 2ab76afda..6aafcff77 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -117,6 +117,8 @@ def _clean_provider_env(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("OPENAI_BASE_URL", raising=False) monkeypatch.delenv("OPENAI_PROJECT_ID", raising=False) monkeypatch.delenv("SKILLSPECTOR_REASONING_EFFORT", raising=False) + monkeypatch.delenv("SKILLSPECTOR_TEMPERATURE", raising=False) + monkeypatch.delenv("SKILLSPECTOR_SEED", raising=False) monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) monkeypatch.delenv("SKILLSPECTOR_MODEL", raising=False) @@ -400,6 +402,25 @@ def fake_chat_anthropic(**kwargs: object) -> dict[str, object]: assert "effort" not in captured + def test_temperature_is_forwarded_without_openai_seed( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + captured: dict[str, object] = {} + + def fake_chat_anthropic(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return kwargs + + monkeypatch.setattr(anthropic_provider_module, "ChatAnthropic", fake_chat_anthropic) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-x") + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0") + monkeypatch.setenv("SKILLSPECTOR_SEED", "42") + + AnthropicProvider().create_chat_model("claude-opus-4-6", max_tokens=123) + + assert captured["temperature"] == 0.0 + assert "seed" not in captured + def test_create_chat_model_returns_none_without_key(self) -> None: # No ANTHROPIC_API_KEY → no client, signalling the caller to fall back. assert AnthropicProvider().create_chat_model("claude-opus-4-6", max_tokens=123) is None @@ -550,6 +571,49 @@ def fake_chat_openai(**kwargs: object) -> dict[str, object]: assert captured["reasoning_effort"] == "provider-specific-value" + def test_sampling_controls_are_forwarded(self, monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} + + def fake_chat_openai(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return kwargs + + monkeypatch.setattr(chat_models, "ChatOpenAI", fake_chat_openai) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", " 0.25 ") + monkeypatch.setenv("SKILLSPECTOR_SEED", "42") + + create_openai_compatible_chat_model( + model="gpt-5.4", + credentials=("sk-x", "http://localhost:1234/v1"), + max_tokens=123, + ) + + assert captured["temperature"] == 0.25 + assert captured["seed"] == 42 + + @pytest.mark.parametrize( + ("name", "value", "message"), + [ + ("SKILLSPECTOR_TEMPERATURE", "warm", "must be a number"), + ("SKILLSPECTOR_TEMPERATURE", "1.1", "must be between 0 and 1"), + ("SKILLSPECTOR_SEED", "4.2", "must be an integer"), + ], + ) + def test_invalid_sampling_control_fails_before_model_construction( + self, + monkeypatch: pytest.MonkeyPatch, + name: str, + value: str, + message: str, + ) -> None: + monkeypatch.setenv(name, value) + with pytest.raises(ValueError, match=message): + create_openai_compatible_chat_model( + model="gpt-5.4", + credentials=("sk-x", "http://localhost:1234/v1"), + max_tokens=123, + ) + class TestProviderSelection: """SKILLSPECTOR_PROVIDER selects which provider answers credentials."""