feat(evaluators): update Luna client to support additional metrics

engine/src/agent_control_engine/core.py

@@ -320,7 +320,7 @@ async def _evaluate_leaf(
                     timeout = DEFAULT_EVALUATOR_TIMEOUT
 
                 result = await asyncio.wait_for(
-                    evaluator.evaluate(data),
+                    evaluator.evaluate_with_context(data, request.step),
                     timeout=timeout,
                 )
         except TimeoutError:

engine/tests/test_core.py

@@ -39,13 +39,15 @@ class SimpleConfig(BaseModel):
 # Shared state for coordination between test evaluators
 _execution_log: list[str] = []
 _blocker_event: asyncio.Event | None = None
+_context_calls: list[tuple[Any, Step]] = []
 
 
 def reset_test_state() -> None:
     """Reset shared test state."""
-    global _execution_log, _blocker_event
+    global _execution_log, _blocker_event, _context_calls
     _execution_log = []
     _blocker_event = asyncio.Event()
+    _context_calls = []
 
 
 class AllowEvaluator(Evaluator[SimpleConfig]):
@@ -163,6 +165,24 @@ async def evaluate(self, data: Any) -> EvaluatorResult:
         return result
 
 
+class ContextEvaluator(Evaluator[SimpleConfig]):
+    """Evaluator that records selector data and full request context."""
+
+    metadata = EvaluatorMetadata(
+        name="test-context",
+        version="1.0.0",
+        description="Records contextual calls",
+    )
+    config_model = SimpleConfig
+
+    async def evaluate(self, data: Any) -> EvaluatorResult:
+        raise AssertionError("engine should call evaluate_with_context")
+
+    async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult:
+        _context_calls.append((data, step))
+        return EvaluatorResult(matched=False, confidence=1.0, message="context received")
+
+
 @dataclass
 class MockControlWithIdentity:
     """Mock control for testing."""
@@ -185,6 +205,7 @@ def setup_test_evaluators():
         BlockerEvaluator,
         SlowEvaluator,
         MetadataEvaluator,
+        ContextEvaluator,
     ]:
         try:
             register_evaluator(evaluator_cls)
@@ -253,6 +274,69 @@ def make_control(
     )
 
 
+@pytest.mark.asyncio
+async def test_context_evaluator_receives_selected_data_and_complete_step() -> None:
+    # Given: a selector targeting output and a step with structured context
+    engine = ControlEngine(
+        [make_control(1, "context", "test-context", action="observe", path="output")]
+    )
+    step = Step(
+        type="llm",
+        name="test-step",
+        input="question",
+        output="answer",
+        context={"conversation_id": "c-1"},
+        tools=[{"name": "search", "description": "Search", "input_schema": {}}],
+        ground_truth="expected answer",
+    )
+
+    # When: evaluating the request through the engine
+    await engine.process(
+        EvaluationRequest(
+            agent_name="00000000-0000-0000-0000-000000000001",
+            step=step,
+            stage="pre",
+        )
+    )
+
+    # Then: selector behavior is unchanged and the complete Step is separate
+    assert _context_calls == [("answer", step)]
+
+
+@pytest.mark.asyncio
+async def test_cached_context_evaluator_handles_concurrent_steps_without_retaining_state() -> None:
+    # Given: one cached evaluator configuration and two independent requests
+    engine = ControlEngine(
+        [make_control(1, "context", "test-context", action="observe", path="input")]
+    )
+    first = Step(type="llm", name="test-step", input="first", ground_truth="one")
+    second = Step(type="llm", name="test-step", input="second", ground_truth="two")
+
+    # When: the requests are evaluated concurrently
+    await asyncio.gather(
+        engine.process(
+            EvaluationRequest(
+                agent_name="00000000-0000-0000-0000-000000000001",
+                step=first,
+                stage="pre",
+            )
+        ),
+        engine.process(
+            EvaluationRequest(
+                agent_name="00000000-0000-0000-0000-000000000001",
+                step=second,
+                stage="pre",
+            )
+        ),
+    )
+
+    # Then: each selected value remains paired with its own full Step
+    assert {(data, step.ground_truth) for data, step in _context_calls} == {
+        ("first", "one"),
+        ("second", "two"),
+    }
+
+
 # =============================================================================
 # Test: Parallel Execution
 # =============================================================================

evaluators/builtin/src/agent_control_evaluators/_base.py

@@ -7,7 +7,7 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar
 
-from agent_control_models import EvaluatorResult
+from agent_control_models import EvaluatorResult, Step
 from agent_control_models.base import BaseModel
 
 if TYPE_CHECKING:
@@ -161,6 +161,23 @@ async def evaluate(self, data: Any) -> EvaluatorResult:
         """
         pass
 
+    async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult:
+        """Evaluate selected data with access to the complete runtime step.
+
+        The default implementation preserves compatibility with evaluators that
+        implement only :meth:`evaluate`. Evaluators must treat ``step`` as
+        immutable request-scoped context because evaluator instances are cached
+        and may be invoked concurrently.
+
+        Args:
+            data: Data extracted by the configured selector.
+            step: Complete runtime step for the current request.
+
+        Returns:
+            EvaluatorResult produced by this evaluator.
+        """
+        return await self.evaluate(data)
+
     def get_timeout_seconds(self) -> float:
         """Get timeout in seconds from config or metadata default."""
         timeout_ms: int = getattr(self.config, "timeout_ms", self.metadata.timeout_ms)

evaluators/builtin/tests/test_base.py

@@ -3,11 +3,12 @@
 Architecture: Evaluators take config at __init__, evaluate() only takes data.
 """
 
-import pytest
 from typing import Any
 
+import pytest
+
 from agent_control_evaluators import Evaluator, EvaluatorConfig, EvaluatorMetadata
-from agent_control_models import EvaluatorResult
+from agent_control_models import EvaluatorResult, Step
 
 
 class MockConfig(EvaluatorConfig):
@@ -106,6 +107,20 @@ async def test_mock_evaluator_evaluate_no_match(self):
 
         assert result.matched is False
 
+    @pytest.mark.asyncio
+    async def test_contextual_evaluation_delegates_to_existing_evaluate(self):
+        """Existing evaluators receive selected data through the default hook."""
+        # Given: an evaluator that implements only evaluate(data)
+        evaluator = MockEvaluator.from_dict({"should_match": True})
+        step = Step(type="llm", name="answer", input="full input")
+
+        # When: the engine-facing contextual hook is called
+        result = await evaluator.evaluate_with_context("selected data", step)
+
+        # Then: the legacy evaluate implementation handles the selected data
+        assert result.matched is True
+        assert result.metadata == {"data": "selected data"}
+
     def test_evaluator_config_stored(self):
         """Test that evaluator stores config."""
         evaluator = MockEvaluator.from_dict({"should_match": True})

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py

@@ -25,13 +25,15 @@
     LunaEvaluator,
     LunaEvaluatorConfig,
     LunaOperator,
+    ScorerInvokeRecord,
     ScorerInvokeRequest,
     ScorerInvokeResponse,
 )
 
 __all__ = [
     "GalileoLunaClient",
     "ScorerInvokeRequest",
+    "ScorerInvokeRecord",
     "ScorerInvokeResponse",
     "LunaEvaluator",
     "LunaEvaluatorConfig",

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/__init__.py

@@ -3,6 +3,7 @@
 from agent_control_evaluator_galileo.luna.client import (
     GalileoLunaClient,
     ScorerInvokeInputs,
+    ScorerInvokeRecord,
     ScorerInvokeRequest,
     ScorerInvokeResponse,
 )
@@ -12,6 +13,7 @@
 __all__ = [
     "GalileoLunaClient",
     "ScorerInvokeInputs",
+    "ScorerInvokeRecord",
     "ScorerInvokeRequest",
     "ScorerInvokeResponse",
     "LunaEvaluatorConfig",

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py

@@ -14,7 +14,7 @@
 from urllib.parse import urlsplit
 
 import httpx
-from agent_control_models import JSONObject, JSONValue
+from agent_control_models import JSONObject, JSONValue, Step
 from pydantic import BaseModel, Field, PrivateAttr, model_validator
 
 logger = logging.getLogger(__name__)
@@ -154,7 +154,18 @@ class ScorerInvokeInputs(BaseModel):
     query: JSONValue = ""
     response: JSONValue = ""
     ground_truth: JSONValue = None
-    tools: JSONValue = None
+    tools: list[JSONObject] | None = None
+
+
+class ScorerInvokeRecord(BaseModel):
+    """Structured runtime record sent alongside legacy scorer inputs."""
+
+    type: str = Field(min_length=1)
+    input: JSONValue = None
+    output: JSONValue = None
+    context: JSONObject | None = None
+    tools: list[JSONObject] | None = None
+    dataset_output: JSONValue = None
 
 
 class ScorerInvokeRequest(BaseModel):
@@ -172,6 +183,7 @@ class ScorerInvokeRequest(BaseModel):
     scorer_version_id: str | None = Field(default=None, min_length=1)
     scorer_label: str | None = Field(default=None, min_length=1)
     inputs: ScorerInvokeInputs
+    record: ScorerInvokeRecord | None = None
     config: JSONObject = Field(default_factory=dict)
 
     @model_validator(mode="after")
@@ -357,6 +369,7 @@ async def invoke(
         scorer_label: str | None = None,
         input: JSONValue = None,
         output: JSONValue = None,
+        step: Step | None = None,
         config: JSONObject | None = None,
         timeout: float = DEFAULT_TIMEOUT_SECS,
         headers: dict[str, str] | None = None,
@@ -369,6 +382,7 @@ async def invoke(
             scorer_label: Optional display/metadata label.
             input: Optional user/system prompt text.
             output: Optional model response text.
+            step: Optional complete runtime step used for structured dual-write.
             config: Optional scorer-specific configuration.
             timeout: Request timeout in seconds.
             headers: Additional request headers.
@@ -390,7 +404,22 @@ async def invoke(
             scorer_version_id=scorer_version_id,
             scorer_label=scorer_label,
             inputs=ScorerInvokeInputs(
-                query="" if input is None else input, response="" if output is None else output
+                query="" if input is None else input,
+                response="" if output is None else output,
+                ground_truth=step.ground_truth if step is not None else None,
+                tools=step.tools if step is not None else None,
+            ),
+            record=(
+                ScorerInvokeRecord(
+                    type=step.type,
+                    input=step.input,
+                    output=step.output,
+                    context=step.context,
+                    tools=step.tools,
+                    dataset_output=step.ground_truth,
+                )
+                if step is not None
+                else None
             ),
             config=config if config is not None else {},
         ).to_dict()

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py

@@ -10,7 +10,7 @@
 
 import httpx
 from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator
-from agent_control_models import EvaluatorResult, JSONValue
+from agent_control_models import EvaluatorResult, JSONValue, Step
 
 from .client import GalileoLunaClient, ScorerInvokeResponse
 from .config import LunaEvaluatorConfig, coerce_number
@@ -198,6 +198,22 @@ async def evaluate(self, data: Any) -> EvaluatorResult:
         Returns:
             EvaluatorResult with local threshold decision and scorer metadata.
         """
+        return await self._evaluate(data, step=None)
+
+    async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult:
+        """Evaluate selected data while dual-writing the complete runtime step.
+
+        Args:
+            data: Data selected by the configured control selector.
+            step: Complete runtime step for structured scorer context.
+
+        Returns:
+            EvaluatorResult with local threshold decision and scorer metadata.
+        """
+        return await self._evaluate(data, step=step)
+
+    async def _evaluate(self, data: Any, *, step: Step | None) -> EvaluatorResult:
+        """Run a Luna evaluation with optional structured runtime context."""
         input_text, output_text = self._prepare_payload(data)
         if not (_has_text(input_text) or _has_text(output_text)):
             return EvaluatorResult(
@@ -209,6 +225,8 @@ async def evaluate(self, data: Any) -> EvaluatorResult:
 
         try:
             scorer_kwargs = self._scorer_kwargs()
+            if step is not None:
+                scorer_kwargs["step"] = step
             response = await self._get_client().invoke(
                 **scorer_kwargs,
                 input=input_text if _has_text(input_text) else None,