diff --git a/README.md b/README.md index 90aeae2..fe996bc 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,7 @@ SessionBat is a Python SDK for recording AI session activity and sending it to SessionBat. It is designed for teams that want to debug and understand what your AI app actually did, including: -- user messages -- assistant responses +- model calls and responses - tool calls - document retrievals @@ -70,8 +69,6 @@ session = client.session( interaction = session.interaction(interaction_id="interaction_123") -interaction.user_message("I am locked out of my account") - interaction.retrieval( query="reset password locked out", documents=[ @@ -93,10 +90,10 @@ interaction.tool_call( metrics={"latency_ms": 117, "http_status": 200}, ) -interaction.assistant_response( +interaction.llm( model="gpt-5.4-mini", - request={"messages": [{"role": "user", "content": "I am locked out of my account"}]}, - response={"text": "I found your account. Use the reset link and follow the email prompt."}, + input={"messages": [{"role": "user", "content": "I am locked out of my account"}]}, + output={"text": "I found your account. Use the reset link and follow the email prompt."}, metrics={"latency_ms": 820, "input_tokens": 142, "output_tokens": 36}, ) ``` @@ -112,7 +109,7 @@ That emits structured events like: "context": {"environment": "prod", "user_id": "user_123", "workspace_id": "ws_456"}, "observation": { "kind": "llm", - "name": "assistant_response" + "name": null } } ``` @@ -154,9 +151,7 @@ delivery before exiting a short-lived process. A `Session` groups interactions. A `Interaction` records completed observations: -- `interaction.user_message(content)` -- `interaction.message(role=..., content=...)` -- `interaction.assistant_response(...)` +- `interaction.llm(...)` - `interaction.tool_call(...)` - `interaction.retrieval(...)` diff --git a/examples/failure_model_error.py b/examples/failure_model_error.py index 89b7354..e331e48 100644 --- a/examples/failure_model_error.py +++ b/examples/failure_model_error.py @@ -21,11 +21,9 @@ def main() -> None: ) interaction = session.interaction(interaction_id="interaction_model_error") - interaction.user_message("What invoices are overdue for Acme Corp?") - - interaction.assistant_response( + interaction.llm( model="gpt-5.4-mini", - request={ + input={ "messages": [ {"role": "user", "content": "What invoices are overdue for Acme Corp?"}, ] diff --git a/examples/failure_retrieval_miss.py b/examples/failure_retrieval_miss.py index e02df68..902ccca 100644 --- a/examples/failure_retrieval_miss.py +++ b/examples/failure_retrieval_miss.py @@ -21,8 +21,6 @@ def main() -> None: ) interaction = session.interaction(interaction_id="interaction_retrieval_miss") - interaction.user_message("How do I export audit logs for SSO users?") - interaction.retrieval( query="export audit logs for sso users", documents=[], @@ -34,23 +32,19 @@ def main() -> None: metrics={"latency_ms": 63, "documents_found": 0}, ) - interaction.assistant_response( + interaction.llm( model="gpt-5.4-mini", - request={ + input={ "messages": [ {"role": "user", "content": "How do I export audit logs for SSO users?"}, ] }, - response={ + output={ "text": "I could not find documentation for exporting audit logs for SSO users.", }, metadata={"provider": "openai"}, metrics={"latency_ms": 704, "input_tokens": 118, "output_tokens": 24}, ) - interaction.user_message( - "Can an admin download SAML access logs anywhere?", - metadata={"sequence": "follow_up"}, - ) if __name__ == "__main__": diff --git a/examples/failure_tool_loop.py b/examples/failure_tool_loop.py index b1c797f..6884fed 100644 --- a/examples/failure_tool_loop.py +++ b/examples/failure_tool_loop.py @@ -21,8 +21,6 @@ def main() -> None: ) interaction = session.interaction(interaction_id="interaction_tool_loop") - interaction.user_message("Why is my workspace usage total wrong?") - interaction.tool_call( tool_name="get_workspace_usage", input={"workspace_id": "ws_999"}, @@ -47,14 +45,14 @@ def main() -> None: metrics={"latency_ms": 139, "http_status": 200, "attempt": 3}, ) - interaction.assistant_response( + interaction.llm( model="gpt-5.4-mini", - request={ + input={ "messages": [ {"role": "user", "content": "Why is my workspace usage total wrong?"}, ] }, - response={ + output={ "text": ( "I checked the usage totals again, but I still cannot determine the discrepancy." ), diff --git a/examples/manual_trace.py b/examples/manual_trace.py index 9d4738e..8db89df 100644 --- a/examples/manual_trace.py +++ b/examples/manual_trace.py @@ -21,8 +21,6 @@ def main() -> None: ) interaction = session.interaction(interaction_id="interaction_123") - interaction.user_message("I am locked out of my account") - interaction.retrieval( query="reset password locked out", documents=[ @@ -48,10 +46,10 @@ def main() -> None: metrics={"latency_ms": 117, "http_status": 200}, ) - interaction.assistant_response( + interaction.llm( model="gpt-5.4-mini", - request={"messages": [{"role": "user", "content": "I am locked out of my account"}]}, - response={"text": "I found your account. Use the reset link and follow the email prompt."}, + input={"messages": [{"role": "user", "content": "I am locked out of my account"}]}, + output={"text": "I found your account. Use the reset link and follow the email prompt."}, metadata={"provider": "openai"}, metrics={"latency_ms": 820, "input_tokens": 142, "output_tokens": 36}, ) diff --git a/src/sessionbat/client.py b/src/sessionbat/client.py index 0b8d297..b4e7431 100644 --- a/src/sessionbat/client.py +++ b/src/sessionbat/client.py @@ -7,8 +7,7 @@ from .models import Envelope, isoformat, new_id, utc_now from .transports import DEFAULT_INGESTION_ENDPOINT, IngestionTransport, Transport -ObservationKind = Literal["message", "llm", "tool", "retrieval"] -MessageRole = Literal["user", "assistant", "system", "tool"] +ObservationKind = Literal["llm", "tool", "retrieval"] def _merge_tags(*tag_sets: list[str] | None) -> list[str]: @@ -145,47 +144,12 @@ class Interaction: tags: list[str] = field(default_factory=list) context: dict[str, Any] = field(default_factory=dict) - def message( - self, - *, - role: MessageRole, - content: Any, - name: str | None = None, - metadata: dict[str, Any] | None = None, - tags: list[str] | None = None, - context: dict[str, Any] | None = None, - ) -> str: - return self._record( - kind="message", - name=name or f"{role}_message", - input={"content": content}, - metadata=_merge_dicts({"role": role}, metadata), - tags=tags, - context=context, - ) - - def user_message( - self, - content: Any, - *, - metadata: dict[str, Any] | None = None, - tags: list[str] | None = None, - context: dict[str, Any] | None = None, - ) -> str: - return self.message( - role="user", - content=content, - metadata=metadata, - tags=tags, - context=context, - ) - - def assistant_response( + def llm( self, *, model: str, - request: dict[str, Any] | None = None, - response: dict[str, Any] | None = None, + input: dict[str, Any] | None = None, + output: dict[str, Any] | None = None, error: dict[str, Any] | None = None, metadata: dict[str, Any] | None = None, metrics: dict[str, Any] | None = None, @@ -194,9 +158,8 @@ def assistant_response( ) -> str: return self._record( kind="llm", - name="assistant_response", - input=request, - output=response, + input=input, + output=output, error=error, metadata=_merge_dicts({"model": model}, metadata), metrics=metrics, @@ -241,7 +204,6 @@ def retrieval( ) -> str: return self._record( kind="retrieval", - name="retrieval", input={"query": query}, output={"documents": documents or []}, error=error, @@ -255,7 +217,7 @@ def _record( self, *, kind: ObservationKind, - name: str, + name: str | None = None, input: dict[str, Any] | None = None, output: dict[str, Any] | None = None, error: dict[str, Any] | None = None, diff --git a/src/sessionbat/langchain.py b/src/sessionbat/langchain.py index 64febaf..379f42c 100644 --- a/src/sessionbat/langchain.py +++ b/src/sessionbat/langchain.py @@ -137,12 +137,6 @@ def on_chat_model_start( extra={"langchain_callback": "on_chat_model_start", "chat_model": True}, kwargs=kwargs, ) - self._record_messages_from_chat_input( - messages, - interaction=interaction, - metadata=callback_metadata, - tags=tags, - ) self._runs[_run_id(run_id)] = _RunState( kind="llm", name=_serialized_name(serialized, "chat_model"), @@ -170,10 +164,10 @@ def on_llm_end( state = self._pop_run(run_id, kind="llm", name="llm", parent_run_id=parent_run_id) response_payload = _llm_response_payload(response) model_metadata = _merge_dicts(state.metadata, metadata) - state.interaction.assistant_response( + state.interaction.llm( model=_extract_model(response=response, metadata=model_metadata), - request=state.input, - response=response_payload, + input=state.input, + output=response_payload, metadata=_merge_dicts( state.metadata, self._callback_metadata( @@ -200,10 +194,10 @@ def on_llm_error( **kwargs: Any, ) -> None: state = self._pop_run(run_id, kind="llm", name="llm", parent_run_id=parent_run_id) - state.interaction.assistant_response( + state.interaction.llm( model=_extract_model(metadata=_merge_dicts(state.metadata, metadata)), - request=state.input, - response=_jsonable(kwargs.get("response")), + input=state.input, + output=_jsonable(kwargs.get("response")), error=_error_payload(error), metadata=_merge_dicts( state.metadata, @@ -493,32 +487,6 @@ def _callback_metadata( def _tags(self, *tag_sets: list[str] | None) -> list[str]: return _merge_tags(self.tags, *tag_sets) - def _record_messages_from_chat_input( - self, - message_batches: list[list[Any]], - *, - interaction: SessionBatInteraction, - metadata: dict[str, Any], - tags: list[str] | None, - ) -> None: - for message in _system_and_user_messages(message_batches): - role = _message_role(message) - if role is None: - continue - interaction.message( - role=role, - content=_message_content(message), - metadata=_merge_dicts( - metadata, - { - "langchain_message_type": _message_type(message), - "langchain_message_id": getattr(message, "id", None), - }, - ), - tags=self._tags(tags), - context=self.context, - ) - def _interaction_for( self, run_id: UUID, @@ -816,45 +784,6 @@ def _document_payload(document: Any) -> dict[str, Any]: return {"content": _jsonable(document)} -def _system_and_user_messages(message_batches: list[list[Any]]) -> list[Any]: - if not message_batches: - return [] - messages = message_batches[-1] - return [message for message in messages if _message_role(message) in {"system", "user"}] - - -def _message_role(message: Any) -> str | None: - message_type = _message_type(message) - if message_type in {"human", "user"}: - return "user" - if message_type == "system": - return "system" - return None - - -def _message_type(message: Any) -> str | None: - message_type = getattr(message, "type", None) - if isinstance(message_type, str): - return message_type - role = getattr(message, "role", None) - if isinstance(role, str): - return role - if isinstance(message, Mapping): - value = message.get("type") or message.get("role") - if isinstance(value, str): - return value - return None - - -def _message_content(message: Any) -> Any: - content = getattr(message, "content", None) - if content is not None: - return _jsonable(content) - if isinstance(message, Mapping) and "content" in message: - return _jsonable(message["content"]) - return _jsonable(message) - - def _jsonable(value: Any, *, _depth: int = 0) -> Any: if _depth > 8: return repr(value) diff --git a/tests/integration/test_langchain.py b/tests/integration/test_langchain.py index 2e93b8b..df3d24a 100644 --- a/tests/integration/test_langchain.py +++ b/tests/integration/test_langchain.py @@ -153,42 +153,24 @@ def test_integrates_with_langchain_runnable_and_fake_chat_model(self) -> None: ) assert result.content == "Use the password reset link." - assert len(self.transport.events) == 3 + assert len(self.transport.events) == 1 assert len({event["session_id"] for event in self.transport.events}) == 1 assert len({event["interaction_id"] for event in self.transport.events}) == 1 - system_event = self.transport.events[0] - system_observation = system_event["observation"] - user_event = self.transport.events[1] - user_observation = user_event["observation"] - llm_event = self.transport.events[2] + llm_event = self.transport.events[0] llm_observation = llm_event["observation"] - assert system_event["type"] == "message" - assert system_observation["kind"] == "message" - assert system_observation["name"] == "system_message" - assert system_observation["input"] == {"content": "You help users recover account access."} - assert system_observation["metadata"]["role"] == "system" - assert system_observation["metadata"]["framework"] == "langchain" - assert system_observation["metadata"]["tenant"] == "acme" - - assert user_event["type"] == "message" - assert user_observation["kind"] == "message" - assert user_observation["name"] == "user_message" - assert user_observation["input"] == {"content": "I am locked out"} - assert user_observation["metadata"]["role"] == "user" - assert user_observation["metadata"]["framework"] == "langchain" - assert user_observation["metadata"]["tenant"] == "acme" - assert llm_event["type"] == "llm" assert llm_observation["kind"] == "llm" - assert llm_observation["name"] == "assistant_response" + assert llm_observation["name"] is None assert llm_observation["output"]["text"] == "Use the password reset link." assert llm_observation["metadata"]["framework"] == "langchain" assert llm_observation["metadata"]["chat_model"] is True assert llm_observation["metadata"]["langchain_serialized_name"] == "FakeSupportChat" assert llm_observation["metadata"]["tenant"] == "acme" assert "messages" in llm_observation["input"] + assert "You help users recover account access." in str(llm_observation["input"]["messages"]) + assert "I am locked out" in str(llm_observation["input"]["messages"]) assert "langchain" in llm_event["tags"] assert "invoke" in llm_event["tags"] @@ -232,16 +214,15 @@ def test_integrates_with_langchain_retrieval_chain(self) -> None: ) assert result.content == "Use the password reset link from the sign-in page." - assert len(self.transport.events) == 4 + assert len(self.transport.events) == 2 assert len({event["session_id"] for event in self.transport.events}) == 1 assert len({event["interaction_id"] for event in self.transport.events}) == 1 retrieval_observation = self.transport.events[0]["observation"] - system_observation = self.transport.events[1]["observation"] - user_observation = self.transport.events[2]["observation"] - llm_observation = self.transport.events[3]["observation"] + llm_observation = self.transport.events[1]["observation"] assert self.transport.events[0]["type"] == "retrieval" + assert self.transport.events[1]["type"] == "llm" assert retrieval_observation["kind"] == "retrieval" assert retrieval_observation["input"] == {"query": "How do I reset my password?"} assert retrieval_observation["metrics"]["documents_found"] == 1 @@ -251,20 +232,14 @@ def test_integrates_with_langchain_retrieval_chain(self) -> None: == "Reset your password from the sign-in page." ) - assert system_observation["kind"] == "message" - assert system_observation["metadata"]["role"] == "system" - assert ( - "Reset your password from the sign-in page." in system_observation["input"]["content"] - ) - - assert user_observation["kind"] == "message" - assert user_observation["input"] == {"content": "How do I reset my password?"} - assert user_observation["metadata"]["role"] == "user" - assert llm_observation["kind"] == "llm" assert llm_observation["output"]["text"] == result.content assert llm_observation["metadata"]["langchain_serialized_name"] == "FakeRagChat" - assert "rag" in self.transport.events[3]["tags"] + assert "Reset your password from the sign-in page." in str( + llm_observation["input"]["messages"] + ) + assert "How do I reset my password?" in str(llm_observation["input"]["messages"]) + assert "rag" in self.transport.events[1]["tags"] def test_uses_langchain_metadata_session_id_when_present(self) -> None: handler = self.client.langchain_callback(tags=["langchain"]) diff --git a/tests/test_client.py b/tests/test_client.py index 0964d82..4bd1db1 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -4,7 +4,6 @@ import threading import time from collections.abc import Iterator -from datetime import datetime from http.server import BaseHTTPRequestHandler, HTTPServer from queue import Queue from urllib.parse import urlsplit @@ -33,44 +32,11 @@ def setup_method(self) -> None: interaction_id="interaction_123", tags=["turn-1"] ) - def test_records_user_message_payload(self) -> None: - observation_id = self.interaction.user_message( - "I am locked out", - tags=["urgent", "support"], - context={"locale": "en-US"}, - metadata={"channel": "chat"}, - ) - - event = self.transport.events[0] - observation = event["observation"] - - assert event["id"] == observation_id - assert event["type"] == "message" - assert event["session_id"] == "thread_123" - assert event["interaction_id"] == "interaction_123" - assert event["tags"] == ["production", "support", "password-reset", "turn-1", "urgent"] - assert event["context"] == { - "environment": "prod", - "workspace_id": "ws_123", - "user_id": "user_123", - "locale": "en-US", - } - assert observation["kind"] == "message" - assert observation["name"] == "user_message" - assert observation["input"] == {"content": "I am locked out"} - assert observation["metadata"] == {"role": "user", "channel": "chat"} - assert observation["output"] is None - assert observation["error"] is None - assert observation["metrics"] == {} - datetime.fromisoformat(event["created_at"]) - datetime.fromisoformat(observation["recorded_at"]) - json.dumps(event) - - def test_records_assistant_response_payload(self) -> None: - self.interaction.assistant_response( + def test_records_llm_payload(self) -> None: + self.interaction.llm( model="gpt-test", - request={"messages": [{"role": "user", "content": "Help"}]}, - response={"text": "Use the reset link."}, + input={"messages": [{"role": "user", "content": "Help"}]}, + output={"text": "Use the reset link."}, metrics={"input_tokens": 10, "output_tokens": 6}, metadata={"provider": "openai"}, ) @@ -78,7 +44,7 @@ def test_records_assistant_response_payload(self) -> None: observation = self.transport.events[0]["observation"] assert observation["kind"] == "llm" - assert observation["name"] == "assistant_response" + assert observation["name"] is None assert observation["input"] == {"messages": [{"role": "user", "content": "Help"}]} assert observation["output"] == {"text": "Use the reset link."} assert observation["metadata"] == {"model": "gpt-test", "provider": "openai"} @@ -115,7 +81,7 @@ def test_records_retrieval_payload(self) -> None: assert event["type"] == "retrieval" assert observation["kind"] == "retrieval" - assert observation["name"] == "retrieval" + assert observation["name"] is None assert observation["input"] == {"query": "reset password"} assert observation["output"] == {"documents": [{"id": "doc_123", "score": 0.93}]} assert observation["metadata"] == {"index": "support_articles"} @@ -138,10 +104,17 @@ def test_records_errors_on_failed_operations(self) -> None: def test_session_records_observations_only_through_interactions(self) -> None: assert not hasattr(self.session, "user_message") + assert not hasattr(self.session, "message") assert not hasattr(self.session, "assistant_response") + assert not hasattr(self.session, "llm") assert not hasattr(self.session, "tool_call") assert not hasattr(self.session, "retrieval") + def test_interaction_does_not_expose_message_observation_apis(self) -> None: + assert not hasattr(self.interaction, "user_message") + assert not hasattr(self.interaction, "message") + assert not hasattr(self.interaction, "assistant_response") + class _RecordingHandler(BaseHTTPRequestHandler): requests: list[dict] = [] diff --git a/tests/test_langchain.py b/tests/test_langchain.py index 9489c8d..4dc0fd0 100644 --- a/tests/test_langchain.py +++ b/tests/test_langchain.py @@ -57,7 +57,7 @@ def test_client_convenience_constructor_does_not_require_session_id(self) -> Non assert isinstance(handler, LangChainCallbackHandler) - def test_records_llm_completion_as_assistant_response(self) -> None: + def test_records_llm_completion(self) -> None: handler = self.session.langchain_callback( tags=["langchain"], metadata={"source": "test"}, @@ -82,7 +82,7 @@ def test_records_llm_completion_as_assistant_response(self) -> None: assert event["tags"] == ["development", "support-bot", "langchain"] assert event["context"] == {"environment": "test", "user_id": "user_123"} assert observation["kind"] == "llm" - assert observation["name"] == "assistant_response" + assert observation["name"] is None assert observation["input"]["prompts"] == ["I am locked out"] assert observation["output"]["text"] == "Use the reset link." assert observation["metadata"]["framework"] == "langchain" @@ -95,7 +95,7 @@ def test_records_llm_completion_as_assistant_response(self) -> None: assert "latency_ms" in observation["metrics"] json.dumps(event) - def test_records_llm_error_as_failed_assistant_response(self) -> None: + def test_records_llm_error(self) -> None: handler = self.session.langchain_callback() run_id = uuid4() @@ -109,6 +109,7 @@ def test_records_llm_error_as_failed_assistant_response(self) -> None: observation = self.transport.events[0]["observation"] assert observation["kind"] == "llm" + assert observation["name"] is None assert observation["metadata"]["model_name"] == "gpt-test" assert observation["error"] == {"type": "RuntimeError", "message": "upstream failed"}