diff --git a/README.md b/README.md index 20f280a..055e30f 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,8 @@ flowchart TD |---|---| | **Frontend** | React 19, Vite, Tailwind CSS, Lucide React | | **State / Auth** | Zustand, `@supabase/supabase-js` | -| **Backend** | FastAPI, Python 3.10+, PyJWT, Tenacity, asyncio | -| **AI / LLM** | LangChain, `langchain-nvidia-ai-endpoints` (NVIDIA NIM) | +| **Backend** | FastAPI, Python 3.10+, PyJWT, Pydantic, Tenacity, asyncio | +| **AI / LLM** | LangChain (LCEL), `langchain-nvidia-ai-endpoints` (NVIDIA NIM) | | **Execution** | Subprocess sandbox (Docker-ready, Data Science env equipped) | | **Database / Auth** | Supabase (PostgreSQL + Row-Level Security) | @@ -114,7 +114,23 @@ Agentloop/ │ ├── api/ │ │ ├── controllers/ # Business logic controllers │ │ └── routes.py # FastAPI router — thin endpoint declarations -│ ├── core/ # DS-STAR Orchestrator, Planner, Coder, Verifier… +│ ├── core/ # DS-STAR orchestrators & core modules +│ │ ├── analyzer/ # Data schema normalizer +│ │ ├── coder/ # Code generation agent +│ │ ├── debugger/ # Error recovery agent +│ │ ├── ds_star_plus/ # Deep Research extensions (ReportWriter, SubQuestion) +│ │ ├── executor/ # Code sandbox execution +│ │ ├── finalizer/ # Insight extraction agent +│ │ ├── planner/ # Multi-step reasoning agent +│ │ ├── retrieval/ # Vector search agent +│ │ ├── router/ # Control flow agent +│ │ ├── verifier/ # Output validation agent +│ │ ├── ds_star_orchestrator.py # Main execution loop +│ │ ├── deep_research_orchestrator.py # DS-STAR+ parallel orchestrator +│ │ ├── token_tracker.py # Token & cost telemetry +│ │ ├── validation.py # State transitions validation +│ │ ├── config.py # Core configuration +│ │ └── llm_client.py # LangChain wrapper │ ├── db/ │ │ ├── create_workspaces.sql # Workspaces table + RLS │ │ ├── create_uploaded_files.sql # File metadata table + RLS @@ -127,7 +143,9 @@ Agentloop/ │ ├── middleware/ │ │ ├── auth.py # JWT dependency (get_current_user / get_optional_user) │ │ └── error_handler.py # Global exception handler -│ ├── models/ # Pydantic request/response schemas +│ ├── models/ +│ │ ├── schemas.py # Pydantic request/response schemas +│ │ └── metrics_schema.py # Evaluation & trace metrics schemas │ ├── services/ │ │ └── supabase_service.py # Supabase client + all DB/Storage ops │ ├── tests/ @@ -188,17 +206,6 @@ Agentloop uses **Supabase JWT authentication** end-to-end: 3. **Backend**: `get_current_user` FastAPI dependency decodes and verifies the JWT using `PyJWT` + `SUPABASE_JWT_SECRET`. Returns `AuthUser(user_id, email, role)`. 4. **Database**: Supabase Row-Level Security policies enforce `auth.uid() = user_id` — users can only access their own rows. -### Protected endpoints -| Method | Path | Auth | -|---|---|---| -| `POST` | `/api/upload` | Optional (anonymous uploads allowed) | -| `POST` | `/api/process` | Optional | -| `POST` | `/api/agent/run` | Optional (user_id stamped when present) | -| `GET` | `/api/agent/runs` | Optional (scoped to user when authenticated) | -| `GET` | `/api/agent/runs/{id}` | Optional | -| `GET` | `/api/workspaces` | **Required** | -| `POST` | `/api/workspaces` | **Required** | - --- ## 🤖 Agent Workflow @@ -211,12 +218,13 @@ Agentloop relies on the **DS-STAR Orchestrator** pattern: 4. **[Coder](docs/coder_agent.md)**: Translates each plan step into self-contained Python code. 5. **[Code Executor](docs/code_executor.md)**: Runs generated code in an isolated sandbox. 6. **[Debugger](docs/debugger_agent.md)**: Surgically corrects failing code blocks. -7. **[Verifier](docs/verifier_agent.md) & [Router](docs/router_agent.md)**: Evaluate output and re-route for retries. -8. **[Finalizer](docs/finalizer_agent.md)**: Converts execution output into clean Markdown insights. +7. **[Verifier](docs/verifier_agent.md)**: Evaluates execution output against user constraints. +8. **[Router](docs/router_agent.md)**: Decides whether to return results or retry (routes back to Planner/Coder). +9. **[Finalizer](docs/finalizer_agent.md)**: Converts execution output into clean Markdown insights. **DS-STAR+** extends with: -- **[SubQuestionGenerator](docs/subquestion_generator_agent.md)**: Decomposes open-ended queries into sub-questions. -- **[ReportWriter](docs/report_writer_agent.md)**: Synthesizes parallel sub-runs into a research report. +- **[SubQuestionGenerator](docs/subquestion_generator_agent.md)**: Decomposes open-ended queries into parallel sub-questions. +- **[ReportWriter](docs/report_writer_agent.md)**: Synthesizes parallel sub-runs into a comprehensive research report. --- @@ -294,35 +302,3 @@ npm run dev cd backend python -m pytest tests/ -v ``` - ---- - -## 🔌 API Reference - -| Method | Path | Auth | Description | -|---|---|---|---| -| `POST` | `/api/upload` | Optional | Upload files to session-scoped cache | -| `POST` | `/api/process` | Optional | Build in-memory document context | -| `POST` | `/api/agent/run` | Optional | Start DS-STAR agent run (SSE stream) | -| `GET` | `/api/agent/runs` | Optional | List past runs (scoped to user) | -| `GET` | `/api/agent/runs/{id}` | Optional | Get a single run by ID | -| `GET` | `/api/workspaces` | **Required** | List authenticated user's workspaces | -| `POST` | `/api/workspaces` | **Required** | Create a new workspace | -| `DELETE` | `/api/clear` | Optional | Wipe session file cache | -| `GET` | `/api/eval/*` | Public | Evaluation metrics endpoints | - ---- - -## 🤝 Contributing - -1. Fork the repository. -2. Create a feature branch: `git checkout -b feature/amazing-feature` -3. Commit changes: `git commit -m 'feat: add amazing feature'` -4. Push: `git push origin feature/amazing-feature` -5. Open a Pull Request. - ---- - -## 📄 License - -MIT License — see [LICENSE](LICENSE) for details. diff --git a/backend/api/controllers/agent_controller.py b/backend/api/controllers/agent_controller.py index 5c19bf1..37f09a7 100644 --- a/backend/api/controllers/agent_controller.py +++ b/backend/api/controllers/agent_controller.py @@ -17,6 +17,11 @@ lightweight background monitor task (polling every 1 s) rather than awaiting http_request.is_disconnected() on every single SSE event. - ARCH-03: All Supabase helper functions are now awaited (async callers). +- WS-01: Workspace auto-hydration: when context is empty and workspace_id is + present, files are loaded from disk into the session cache so the orchestrator + always has data — fixing the root cause of the "files disappear" bug where + workspace files shown in the UI (fetched from Supabase) were never put through + /process and were therefore invisible to the backend session cache. """ import asyncio @@ -27,8 +32,6 @@ from fastapi import Request -from eval.eval_logger import EvalLogger - from core.config import MAX_AGENT_ROUNDS from core.ds_star_orchestrator import DsStarOrchestrator from services.upload_service import clear_file_cache @@ -46,7 +49,6 @@ def _get_orchestrator( - max_rounds: Optional[int], model: Optional[str], coder_model: Optional[str], temperature: Optional[float], @@ -55,14 +57,14 @@ def _get_orchestrator( Orchestrators are keyed by (model, coder_model, temperature) since those determine which ChatNVIDIA instances are built inside. ``max_rounds`` is - not part of the key because it is per-request configurable and is applied - to the orchestrator's ``_max_rounds`` attribute directly. + NOT part of the key and NOT mutated on the cached instance — it is + resolved at call-time and forwarded into ``orchestrator.run()`` directly + (P1-01 fix: eliminates the shared-instance data race). DsStarOrchestrator.run() stores all per-run state in local variables, so sharing instances across requests is safe. Args: - max_rounds: Per-request round limit. model: Reasoning LLM model override. coder_model: Code-generation LLM model override. temperature: Sampling temperature override. @@ -73,7 +75,6 @@ def _get_orchestrator( cache_key: _OrchestratorKey = (model, coder_model, temperature) if cache_key not in _orchestrator_cache: _orchestrator_cache[cache_key] = DsStarOrchestrator( - max_rounds=max_rounds, model=model, coder_model=coder_model, temperature=temperature, @@ -82,10 +83,92 @@ def _get_orchestrator( "[AgentController] Orchestrator created — model=%s, coder=%s, temp=%s", model, coder_model, temperature, ) - orchestrator = _orchestrator_cache[cache_key] - # Apply per-request max_rounds without invalidating the cache - orchestrator._max_rounds = max_rounds or MAX_AGENT_ROUNDS - return orchestrator + return _orchestrator_cache[cache_key] + + +# --------------------------------------------------------------------------- +# WS-01: Workspace auto-hydration helper +# --------------------------------------------------------------------------- + +async def _hydrate_context_from_workspace( + workspace_id: str, + session_id: str, +) -> Dict[str, Any]: + """Loads workspace files from disk into the session cache and returns context. + + Called when ``handle_agent_run`` receives an empty context but a + ``workspace_id`` is set. This happens when the frontend loads files from + Supabase (workspace view) and starts a run without going through ``/process`` + — the in-memory session cache is empty even though files exist on disk. + + Strategy: + 1. Scan ``/workspace/{workspace_id}/`` for data files. + 2. Read each file's bytes into the session cache (``_FILE_CACHE``). + 3. Run the synchronous ``process_documents`` to extract schema context. + 4. Return the resulting ``combined_extractions`` dict. + + Args: + workspace_id: UUID of the workspace whose files to load. + session_id: Session bucket to populate in the file cache. + + Returns: + A context dict compatible with what ``/process`` would return, + or an empty dict if no files are found. + """ + import os # pylint: disable=import-outside-toplevel + from services.upload_service import _FILE_CACHE, _CACHE_LOCK # pylint: disable=import-outside-toplevel + from services.process_service import process_documents # pylint: disable=import-outside-toplevel + + workspace_base = os.environ.get("WORKSPACE_FILES_DIR", "/workspace") + workspace_dir = os.path.join(workspace_base, workspace_id) + + if not os.path.isdir(workspace_dir): + logger.warning( + "[AgentController] Workspace dir not found: %s — cannot auto-hydrate.", + workspace_dir, + ) + return {} + + # Gather all files from the workspace directory + loaded_names = [] + for filename in os.listdir(workspace_dir): + file_path = os.path.join(workspace_dir, filename) + if not os.path.isfile(file_path): + continue + try: + with open(file_path, "rb") as fh: + content = fh.read() + with _CACHE_LOCK: + _FILE_CACHE.setdefault(session_id, {})[filename] = content + loaded_names.append(filename) + logger.info( + "[AgentController] WS-01: hydrated %s (%d bytes) → session=%s", + filename, len(content), session_id[:8], + ) + except OSError as exc: + logger.warning( + "[AgentController] WS-01: could not read %s: %s", file_path, exc + ) + + if not loaded_names: + return {} + + # Build processing context synchronously (CSV/PDF parsing is blocking) + loop = asyncio.get_running_loop() + try: + context = await loop.run_in_executor( + None, process_documents, loaded_names, session_id + ) + logger.info( + "[AgentController] WS-01: auto-hydrated %d file(s) from workspace %s.", + len(loaded_names), workspace_id[:8], + ) + return context + except Exception as exc: # pylint: disable=broad-except + logger.warning( + "[AgentController] WS-01: process_documents failed during hydration: %s", exc + ) + return {} # --------------------------------------------------------------------------- @@ -153,19 +236,56 @@ async def handle_agent_run( """ run_id = uuid.uuid4().hex _session_id = session_id or "__anon__" + _max_rounds = max_rounds or MAX_AGENT_ROUNDS + + # WS-01: Auto-hydrate context from workspace disk when the session cache is + # empty. This happens when the frontend loads workspace files from Supabase + # and starts a run without going through /process (files are shown in the UI + # but never put in the server-side session cache). + _combined = context.get("combined_extractions", {}) + if not _combined and workspace_id: + logger.info( + "[AgentController] WS-01: context is empty but workspace_id=%s — hydrating.", + workspace_id[:8], + ) + hydrated = await _hydrate_context_from_workspace(workspace_id, _session_id) + if hydrated: + context = { + **context, + "combined_extractions": hydrated.get("combined_extractions", {}), + "files_processed": hydrated.get("files_processed", 0), + } + logger.info( + "[AgentController] WS-01: context hydrated with %d file(s).", + context.get("files_processed", 0), + ) # ARCH-01: use cached orchestrator instead of creating a new one per request - orchestrator = _get_orchestrator(max_rounds, model, coder_model, temperature) + # P1-01: max_rounds is resolved here and passed into run(), NOT mutated on + # the shared cached instance, eliminating the concurrent-request data race. + orchestrator = _get_orchestrator(model, coder_model, temperature) # Persist new run row — non-blocking await _try_create_run(run_id, _session_id, query, context, workspace_id, user_id) - # Eval sidecar — passive observer, zero orchestration changes - eval_logger = EvalLogger(run_id=run_id, query=query) - # Emit run_id to the frontend immediately yield f"data: {json.dumps({'event': 'run_started', 'payload': {'run_id': run_id}})}\n\n" + # BUG 4 fix: Emit any parse_warnings accumulated during /process as SSE + # warning events *before* the orchestrator loop starts. This gives the + # user visibility into near-empty files (blank PDFs, image-only scans) + # at the earliest possible moment — before any LLM call is made. + for pw in context.get("parse_warnings", []): + warn_payload = json.dumps({ + "event": "warning", + "payload": { + "message": pw.get("message", "A file produced minimal extractable content."), + "filename": pw.get("filename", ""), + "source": "parse_warning", + }, + }) + yield f"data: {warn_payload}\n\n" + # PERF-03: Set up disconnect monitoring via asyncio.Event disc_event = asyncio.Event() monitor_task: Optional[asyncio.Task] = None @@ -180,6 +300,8 @@ async def handle_agent_run( context, run_id=run_id, session_id=_session_id, + max_rounds=_max_rounds, + workspace_id=workspace_id, ): # PERF-03: O(1) check — no await, no syscall per event if disc_event.is_set(): @@ -189,8 +311,6 @@ async def handle_agent_run( await _try_update_run(run_id, {}, status="failed") return - eval_logger.ingest(event) # ← eval sidecar: passive observation - payload = json.dumps(event, default=str) yield f"data: {payload}\n\n" @@ -201,8 +321,16 @@ async def handle_agent_run( await _try_update_run(run_id, event_payload, status="completed") elif event_type == "error": await _try_update_run(run_id, event_payload, status="failed") - elif event_type == "metrics": - await _try_persist_metrics(run_id, event_payload) + + except asyncio.CancelledError: + # P2-01 fix: CancelledError inherits from BaseException, not Exception. + # FastAPI cancels the streaming task on client disconnect — we must + # catch it explicitly to mark the run as failed before re-raising. + logger.info( + "[AgentController] Stream cancelled (client disconnect?) — run_id=%s", run_id + ) + await _try_update_run(run_id, {}, status="failed") + raise # re-raise so FastAPI can clean up the response properly except Exception as exc: # pylint: disable=broad-except error_event = json.dumps({ @@ -216,7 +344,6 @@ async def handle_agent_run( finally: if monitor_task is not None: monitor_task.cancel() - await eval_logger.finalize() # ← eval sidecar: flush to Supabase # File cache is retained across runs — users can run multiple queries on the # same uploaded data without re-uploading. Cleanup is handled by: # • Explicit DELETE /api/clear (user action via handleClearAll) @@ -286,28 +413,3 @@ async def _try_update_run( ) except Exception as exc: # pylint: disable=broad-except logger.warning("[AgentController] Could not persist run result: %s", exc) - - -async def _try_persist_metrics(run_id: str, metrics_payload: Dict[str, Any]) -> None: - """Attempts to persist run evaluation metrics to Supabase. - - Args: - run_id: Unique run identifier. - metrics_payload: The ``payload`` dict from the ``metrics`` SSE event. - """ - try: - from services.supabase_service import update_agent_run_metrics # pylint: disable=import-outside-toplevel - await update_agent_run_metrics( - run_id=run_id, - metrics=metrics_payload.get("metrics", {}), - total_run_ms=metrics_payload.get("total_run_ms", 0), - complexity=metrics_payload.get("complexity", "easy"), - ) - logger.info( - "[AgentController] Metrics persisted for run_id=%s, complexity=%s, total_ms=%d", - run_id, - metrics_payload.get("complexity", "easy"), - metrics_payload.get("total_run_ms", 0), - ) - except Exception as exc: # pylint: disable=broad-except - logger.warning("[AgentController] Could not persist run metrics: %s", exc) diff --git a/backend/api/controllers/process_controller.py b/backend/api/controllers/process_controller.py index 60bbeb2..0321f9e 100644 --- a/backend/api/controllers/process_controller.py +++ b/backend/api/controllers/process_controller.py @@ -2,9 +2,14 @@ Handles request-level orchestration for the /process endpoint: resolves file paths and delegates to the process service. + +Multi-file extension: + ``get_schema_context`` queries the workspace_files table and runs + schema_merger to produce a MultiFileContext for the agent pipeline. + This is called by the orchestrator when a workspace_id is available. """ -from typing import List, Dict, Any +from typing import Dict, Any, List from services.process_service import process_documents from services.upload_service import clear_file_cache @@ -44,3 +49,30 @@ def handle_clear(session_id: str = "__anon__") -> Dict[str, Any]: """ clear_file_cache(session_id=session_id) return {"status": "success", "message": "In-memory cache completely wiped."} + + +async def get_schema_context(workspace_id: str, user_id: str): + """Builds a MultiFileContext from the workspace_files table. + + Fetches all workspace_files rows for the given workspace and user, + ordered by upload_order ASC, then delegates to schema_merger to infer + join candidates from the stored schema_json metadata. + + Args: + workspace_id: UUID of the target workspace. + user_id: Authenticated user UUID for RLS enforcement. + + Returns: + MultiFileContext with FileSchema objects and join_candidates. + If no files are found, returns an empty MultiFileContext. + """ + from services.supabase_service import list_workspace_files # pylint: disable=import-outside-toplevel + from services.schema_merger import merge_schemas # pylint: disable=import-outside-toplevel + + file_metas = await list_workspace_files(workspace_id=workspace_id, user_id=user_id) + + if not file_metas: + from models.multi_file_context import MultiFileContext # pylint: disable=import-outside-toplevel + return MultiFileContext(files=[], join_candidates=[]) + + return await merge_schemas(file_metas) diff --git a/backend/api/controllers/research_controller.py b/backend/api/controllers/research_controller.py index e230495..ae99bc0 100644 --- a/backend/api/controllers/research_controller.py +++ b/backend/api/controllers/research_controller.py @@ -23,6 +23,8 @@ import uuid from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple +from fastapi import Request + from core.deep_research_orchestrator import DeepResearchOrchestrator logger = logging.getLogger("uvicorn.info") @@ -37,7 +39,6 @@ def _get_research_orchestrator( - max_rounds: Optional[int], model: Optional[str], coder_model: Optional[str], temperature: Optional[float], @@ -46,14 +47,10 @@ def _get_research_orchestrator( """Returns a cached DeepResearchOrchestrator for the given configuration. Keyed by (model, coder_model, temperature, max_workers). Per-request - ``max_rounds`` is applied directly to the cached instance's ``_max_rounds`` - attribute without invalidating the cache. - - DeepResearchOrchestrator.run() stores all per-run state in local variables, - so sharing instances across requests is safe. + ``max_rounds`` is forwarded into ``orchestrator.run()`` rather than + mutated on the cached instance (P1-01 fix: eliminates data race). Args: - max_rounds: Per-request round limit for sub-question DS-STAR runs. model: Reasoning LLM model override. coder_model: Code-generation LLM model override. temperature: Sampling temperature override. @@ -66,7 +63,6 @@ def _get_research_orchestrator( cache_key: _ResearchOrchestratorKey = (model, coder_model, temperature, max_workers) if cache_key not in _research_orchestrator_cache: _research_orchestrator_cache[cache_key] = DeepResearchOrchestrator( - max_rounds=max_rounds, model=model, coder_model=coder_model, temperature=temperature, @@ -76,9 +72,31 @@ def _get_research_orchestrator( "[ResearchController] Orchestrator created — model=%s, workers=%s", model, max_workers, ) - orchestrator = _research_orchestrator_cache[cache_key] - orchestrator._max_rounds = max_rounds or DS_STAR_PLUS_MAX_ROUNDS - return orchestrator + return _research_orchestrator_cache[cache_key] + + +# --------------------------------------------------------------------------- +# PERF-03: Background disconnect monitor (mirrors agent_controller) +# --------------------------------------------------------------------------- + +async def _monitor_disconnect( + http_request: Request, + disc_event: asyncio.Event, + poll_interval: float = 1.0, +) -> None: + """Sets ``disc_event`` when the HTTP client disconnects. + + P2-02 fix: deep-research runs can now be aborted on client disconnect + the same way DS-STAR runs are handled in agent_controller. + """ + while not disc_event.is_set(): + try: + if await http_request.is_disconnected(): + disc_event.set() + return + except Exception: # pylint: disable=broad-except + return # Transport gone — treat as disconnected + await asyncio.sleep(poll_interval) # --------------------------------------------------------------------------- @@ -96,6 +114,7 @@ async def handle_research_run( max_workers: Optional[int] = None, user_id: Optional[str] = None, workspace_id: Optional[str] = None, + http_request: Optional[Request] = None, ) -> AsyncGenerator[str, None]: """Streams DS-STAR+ research events as Server-Sent Events. @@ -110,31 +129,89 @@ async def handle_research_run( max_workers: Override for max parallel DS-STAR sub-runs. user_id: Authenticated user ID. workspace_id: Optional workspace scope. + http_request: FastAPI Request object — used to detect early client + disconnection via a background monitor task (P2-02 fix). Yields: SSE-formatted ``data: \\n\\n`` lines. """ report_id = uuid.uuid4().hex _session_id = session_id or "__anon__" + from core.config import DS_STAR_PLUS_MAX_ROUNDS # pylint: disable=import-outside-toplevel + _max_rounds = max_rounds or DS_STAR_PLUS_MAX_ROUNDS # ARCH-01: use cached orchestrator + # P1-01 fix: max_rounds forwarded to run(), NOT mutated on cached instance. orchestrator = _get_research_orchestrator( - max_rounds, model, coder_model, temperature, max_workers + model, coder_model, temperature, max_workers ) + # WS-01: Auto-hydrate context from workspace disk when the session cache is + # empty. This mirrors the same fix in agent_controller.py — without it, + # DS-STAR+ deep research runs silently get empty context when files were + # uploaded via the workspace path and the session cache was evicted. + _combined = context.get("combined_extractions", {}) + if not _combined and workspace_id: + logger.info( + "[ResearchController] WS-01: context is empty but workspace_id=%s — hydrating.", + workspace_id[:8], + ) + from api.controllers.agent_controller import _hydrate_context_from_workspace # pylint: disable=import-outside-toplevel + hydrated = await _hydrate_context_from_workspace(workspace_id, _session_id) + if hydrated: + context = { + **context, + "combined_extractions": hydrated.get("combined_extractions", {}), + "files_processed": hydrated.get("files_processed", 0), + } + logger.info( + "[ResearchController] WS-01: context hydrated with %d file(s).", + context.get("files_processed", 0), + ) + # Persist report row before streaming starts await _try_create_report(report_id, _session_id, query, context, workspace_id, user_id) # Emit report_id to frontend immediately yield f"data: {json.dumps({'event': 'report_started', 'payload': {'report_id': report_id}})}\n\n" + # BUG 4 fix: Emit parse_warnings from /process as SSE warning events before + # any sub-question orchestration begins. Mirrors agent_controller behaviour. + for pw in context.get("parse_warnings", []): + warn_payload = json.dumps({ + "event": "warning", + "payload": { + "message": pw.get("message", "A file produced minimal extractable content."), + "filename": pw.get("filename", ""), + "source": "parse_warning", + }, + }) + yield f"data: {warn_payload}\n\n" + + # P2-02 fix: Set up disconnect monitoring via asyncio.Event + disc_event = asyncio.Event() + monitor_task: Optional[asyncio.Task] = None + if http_request is not None: + monitor_task = asyncio.create_task( + _monitor_disconnect(http_request, disc_event) + ) + sub_questions_created = False try: # ARCH-05: pass session_id into run() so sub-question executors use correct bucket async for event in orchestrator.run( - query, context, report_id=report_id, session_id=_session_id + query, context, report_id=report_id, session_id=_session_id, + max_rounds=_max_rounds, ): + # P2-02: O(1) disconnect check before each SSE event + if disc_event.is_set(): + logger.info( + "[ResearchController] Client disconnected — aborting report_id=%s", + report_id, + ) + await _try_fail_report(report_id) + return payload = json.dumps(event, default=str) yield f"data: {payload}\n\n" @@ -185,6 +262,16 @@ async def handle_research_run( elif event_type == "error": await _try_fail_report(report_id) + except asyncio.CancelledError: + # P2-01 fix: CancelledError inherits from BaseException, not Exception. + # Must be caught explicitly so the report is marked failed on disconnect. + logger.info( + "[ResearchController] Stream cancelled (client disconnect?) — report_id=%s", + report_id, + ) + await _try_fail_report(report_id) + raise + except Exception as exc: # pylint: disable=broad-except error_event = json.dumps({ "event": "error", @@ -199,6 +286,8 @@ async def handle_research_run( ) finally: + if monitor_task is not None: + monitor_task.cancel() yield "data: {\"event\": \"stream_end\", \"payload\": {}}\n\n" diff --git a/backend/api/controllers/upload_controller.py b/backend/api/controllers/upload_controller.py index 3b8235e..6de3552 100644 --- a/backend/api/controllers/upload_controller.py +++ b/backend/api/controllers/upload_controller.py @@ -2,10 +2,21 @@ Handles request-level orchestration for file upload: validates metadata, delegates persistence to upload_service, -and builds the structured UploadResponse. +writes workspace_files rows to Supabase, and returns a +MultiFileContext with inferred join candidates. + +Multi-file changes (Phase 9): + - Sequential per-file processing with upload_order tracking. + - Parser-based schema_json + row_count extraction per file. + - workspace_files DB insert after each accepted file. + - schema_merger called after all files to build MultiFileContext. + - DELETE handler for workspace file removal with schema refresh. """ -from typing import List, Optional +import logging +import math +import os +from typing import Any, Dict, List, Optional from fastapi import UploadFile @@ -13,6 +24,269 @@ from models.schemas import UploadResponse, FileStatusItem from services.upload_service import save_upload_file +logger = logging.getLogger("uvicorn.error") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _get_parser_for(file_type: str): + """Returns the appropriate synchronous parser function for a file type. + + Args: + file_type: Lowercase file extension string (e.g. 'csv', 'xlsx'). + + Returns: + A callable ``parse_*(file_name, file_content_bytes) -> dict`` or None + if no structured parser is available for this type. + """ + if file_type == "csv": + from services.parsers.csv_parser import parse_csv # pylint: disable=import-outside-toplevel + return parse_csv + if file_type in ("xlsx", "xls"): + from services.parsers.excel_parser import parse_excel # pylint: disable=import-outside-toplevel + return parse_excel + if file_type == "parquet": + from services.parsers.parquet_parser import parse_parquet # pylint: disable=import-outside-toplevel + return parse_parquet + if file_type == "json": + from services.parsers.json_parser import parse_json # pylint: disable=import-outside-toplevel + return parse_json + if file_type in ("md", "markdown"): + from services.parsers.md_parser import parse_md # pylint: disable=import-outside-toplevel + return parse_md + return None + + +def _extract_schema_json(parsed: Dict[str, Any]) -> Dict[str, Any]: + """Extracts the schema_json payload from a UnifiedDocumentContext dict. + + We store the ``metadata`` block as schema_json because it contains the + column list, dtypes, and sample rows that schema_merger needs. + + Args: + parsed: Dict returned by a parser function. + + Returns: + The metadata sub-dict, or empty dict if not present. + """ + return parsed.get("metadata") or {} + + +def _resolve_file_path(workspace_id: str, filename: str) -> str: + """Builds the canonical on-disk path for an uploaded workspace file. + + Args: + workspace_id: Workspace UUID used as the directory name. + filename: Original filename (sanitised by save_upload_file). + + Returns: + Absolute path string. + """ + base_dir = os.environ.get("WORKSPACE_FILES_DIR", "/workspace") + return os.path.join(base_dir, workspace_id, filename) + + +def _json_safe(obj: Any) -> Any: + """Recursively replaces non-JSON-compliant floats (NaN, Inf) with None. + + Acts as a safety net at the DB-insert boundary in case a parser + produces dicts containing ``float('nan')`` or ``float('inf')``. + The JSON spec does not permit these values and supabase-py's + ``json.dumps()`` will raise ``ValueError`` if they are present. + + Args: + obj: Any Python object (dict, list, scalar). + + Returns: + The same structure with offending floats replaced by None. + """ + if isinstance(obj, float): + if math.isnan(obj) or math.isinf(obj): + return None + return obj + if isinstance(obj, dict): + return {k: _json_safe(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_json_safe(item) for item in obj] + return obj + + +# --------------------------------------------------------------------------- +# Multi-file workspace upload handler +# --------------------------------------------------------------------------- + +async def handle_workspace_upload( + files: List[UploadFile], + workspace_id: str, + user_id: str, + session_id: str = "__anon__", +) -> Dict[str, Any]: + """Orchestrates validation, persistence, and schema inference for a batch of files. + + Processing is sequential (not concurrent) so upload_order is strictly + monotonic. For each accepted file: + 1. Validate extension / MIME / magic bytes. + 2. Save to disk at /workspace/{workspace_id}/{filename}. + 3. Parse to extract schema_json and row_count. + 4. INSERT into workspace_files with the next upload_order. + + After all files are processed: + 5. Fetch all workspace_files rows for the workspace. + 6. Run schema_merger to build a MultiFileContext. + + Args: + files: Multipart file payloads from the request. + workspace_id: Target workspace UUID. + user_id: Authenticated user UUID (used for ownership columns). + session_id: Session identifier for the in-memory file cache. + + Returns: + Dict with keys: + - ``accepted_files``: List of FileStatusItem dicts (successes). + - ``rejected_files``: List of FileStatusItem dicts (failures). + - ``multi_file_context``: MultiFileContext dict (join candidates etc.) + """ + from services.supabase_service import ( # pylint: disable=import-outside-toplevel + insert_workspace_file, + count_workspace_files, + list_workspace_files, + ) + from services.schema_merger import merge_schemas # pylint: disable=import-outside-toplevel + + accepted: List[FileStatusItem] = [] + rejected: List[FileStatusItem] = [] + + for file in files: + if not file.filename: + continue + + # Extension, MIME & magic-byte validation + meta_issue = await validate_file_metadata(file) + if meta_issue: + rejected.append( + FileStatusItem(filename=file.filename, status="error", reason=meta_issue) + ) + continue + + file_ext = file.filename.rsplit(".", 1)[-1].lower() if "." in file.filename else "" + # Normalise xlsx variants + file_type = "xlsx" if file_ext == "xls" else file_ext + + try: + # Read file bytes (seek back to 0 after validation consumed the stream) + await file.seek(0) + file_bytes = await file.read() + + # Save to session-scoped in-memory cache (for FileAnalyzerAgent LLM path) + await file.seek(0) + await save_upload_file( + file, + session_id=session_id, + user_id=user_id, + workspace_id=workspace_id, + ) + + # Resolve the on-disk path used by the Coder agent + file_path = _resolve_file_path(workspace_id, file.filename) + + # Save bytes to disk (create workspace dir if needed) + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "wb") as fh: + fh.write(file_bytes) + + # Parse to extract schema_json and row_count + parser = _get_parser_for(file_type) + schema_json: Dict[str, Any] = {} + row_count: int = 0 + + if parser is not None: + try: + parsed = parser(file.filename, file_bytes) + schema_json = _extract_schema_json(parsed) + row_count = schema_json.get("row_count") or 0 + except Exception as parse_exc: # pylint: disable=broad-except + logger.warning( + "[UploadController] Parser failed for %s (%s); " + "schema_json will be empty.", + file.filename, + parse_exc, + ) + + # Determine upload_order = existing row count + 1 + existing_count = await count_workspace_files( + workspace_id=workspace_id, + user_id=user_id, + ) + upload_order = existing_count + 1 + + # Insert workspace_files row + db_record = { + "workspace_id": workspace_id, + "user_id": user_id, + "file_name": file.filename, + "file_path": file_path, + "file_type": file_type, + "file_size": len(file_bytes), + "row_count": row_count if row_count > 0 else None, + "schema_json": _json_safe(schema_json), + "upload_order": upload_order, + } + try: + await insert_workspace_file(db_record) + except Exception as db_exc: # pylint: disable=broad-except + logger.warning( + "[UploadController] workspace_files insert failed for %s: %s", + file.filename, + db_exc, + ) + # Don't abort — file is on disk and in cache; the session still works. + + accepted.append( + FileStatusItem( + filename=file.filename, + status="success", + reason=f"Uploaded (order={upload_order}, rows={row_count}).", + ) + ) + + except ValueError as ve: + rejected.append( + FileStatusItem(filename=file.filename, status="error", reason=str(ve)) + ) + except Exception as exc: # pylint: disable=broad-except + rejected.append( + FileStatusItem( + filename=file.filename, + status="error", + reason=f"Stream error: {str(exc)}", + ) + ) + + # Build MultiFileContext from ALL current workspace_files rows + all_metas = await list_workspace_files(workspace_id=workspace_id, user_id=user_id) + multi_file_context = await merge_schemas(all_metas) + + logger.info( + "[UploadController] workspace=%s accepted=%d rejected=%d files_total=%d candidates=%d", + workspace_id[:8], + len(accepted), + len(rejected), + len(all_metas), + len(multi_file_context.join_candidates), + ) + + return { + "accepted_files": accepted, + "rejected_files": rejected, + "multi_file_context": multi_file_context.model_dump(), + } + + +# --------------------------------------------------------------------------- +# Legacy single-file upload handler (backward compat — unchanged) +# --------------------------------------------------------------------------- async def handle_upload( files: List[UploadFile], @@ -22,11 +296,16 @@ async def handle_upload( ) -> UploadResponse: """Orchestrates validation and persistence for a batch of uploaded files. + This is the existing /upload endpoint handler kept for backward + compatibility. It saves files to the session-scoped in-memory cache + only and does NOT write to workspace_files or return a MultiFileContext. + Args: files: Multipart file payloads from the request. - session_id: Session identifier used to bucket files in the in-memory - cache. Ensures files from different users never collide. + session_id: Session identifier for the in-memory cache. user_id: Optional authenticated user identifier. + workspace_id: Optional workspace UUID (passed to save_upload_file + for storage path but does NOT trigger workspace_files inserts). Returns: UploadResponse: Lists of accepted and rejected FileStatusItems. @@ -38,7 +317,6 @@ async def handle_upload( if not file.filename: continue - # Extension, MIME & magic-byte validation (MIN-04: now async) meta_issue = await validate_file_metadata(file) if meta_issue: rejected.append( @@ -47,7 +325,12 @@ async def handle_upload( continue try: - await save_upload_file(file, session_id=session_id, user_id=user_id, workspace_id=workspace_id) + await save_upload_file( + file, + session_id=session_id, + user_id=user_id, + workspace_id=workspace_id, + ) accepted.append( FileStatusItem( filename=file.filename, @@ -59,13 +342,103 @@ async def handle_upload( rejected.append( FileStatusItem(filename=file.filename, status="error", reason=str(ve)) ) - except Exception as e: # pylint: disable=broad-except + except Exception as exc: # pylint: disable=broad-except rejected.append( FileStatusItem( filename=file.filename, status="error", - reason=f"Stream error: {str(e)}", + reason=f"Stream error: {str(exc)}", ) ) return UploadResponse(accepted_files=accepted, rejected_files=rejected) + + +# --------------------------------------------------------------------------- +# DELETE workspace file handler +# --------------------------------------------------------------------------- + +async def handle_delete_workspace_file( + file_id: str, + workspace_id: str, + user_id: str, +) -> Dict[str, Any]: + """Removes a file from the workspace and refreshes the join context. + + Steps: + 1. Fetch the workspace_files row to get file_path (ownership check + is embedded in the Supabase delete call via eq(user_id)). + 2. Delete the row from workspace_files. + 3. Delete the physical file from disk (best-effort). + 4. Re-run schema_merger on remaining files. + 5. Return the updated MultiFileContext. + + Args: + file_id: UUID of the workspace_files row to delete. + workspace_id: Workspace UUID for ownership verification. + user_id: Authenticated user UUID. + + Returns: + Dict with keys: + - ``deleted``: bool — True if a row was removed. + - ``multi_file_context``: Updated MultiFileContext dict. + + Raises: + ValueError: If the file_id is not found or not owned by user. + """ + from services.supabase_service import ( # pylint: disable=import-outside-toplevel + list_workspace_files, + delete_workspace_file, + ) + from services.schema_merger import merge_schemas # pylint: disable=import-outside-toplevel + + # Fetch current rows to get file_path before deletion + all_rows = await list_workspace_files(workspace_id=workspace_id, user_id=user_id) + target_row = next((r for r in all_rows if r.get("id") == file_id), None) + + if not target_row: + raise ValueError( + f"workspace_file '{file_id}' not found in workspace '{workspace_id}' " + f"or not owned by this user." + ) + + # Delete from DB + deleted = await delete_workspace_file( + file_id=file_id, + workspace_id=workspace_id, + user_id=user_id, + ) + + # Best-effort physical file deletion + file_path = target_row.get("file_path", "") + if file_path and os.path.isfile(file_path): + try: + os.remove(file_path) + logger.info( + "[UploadController] Deleted physical file: %s", + file_path, + ) + except OSError as os_exc: + logger.warning( + "[UploadController] Could not delete physical file %s: %s", + file_path, + os_exc, + ) + + # Refresh MultiFileContext from remaining rows + remaining_rows = await list_workspace_files(workspace_id=workspace_id, user_id=user_id) + multi_file_context = await merge_schemas(remaining_rows) + + logger.info( + "[UploadController] Deleted file_id=%s from workspace=%s; " + "%d file(s) remaining, %d candidate(s).", + file_id[:8], + workspace_id[:8], + len(remaining_rows), + len(multi_file_context.join_candidates), + ) + + return { + "deleted": deleted, + "multi_file_context": multi_file_context.model_dump(), + } diff --git a/backend/api/routes.py b/backend/api/routes.py index cb62c2b..312cfb1 100644 --- a/backend/api/routes.py +++ b/backend/api/routes.py @@ -27,6 +27,7 @@ - user_id from the authenticated token is forwarded to service layers. """ +import asyncio import time as _time import io import zipfile @@ -40,9 +41,8 @@ from api.controllers.agent_controller import handle_agent_run from api.controllers.process_controller import handle_clear, handle_process -from api.controllers.upload_controller import handle_upload +from api.controllers.upload_controller import handle_upload, handle_workspace_upload, handle_delete_workspace_file from api.controllers.research_controller import handle_research_run -from eval.eval_routes import eval_router from core.deep_research_orchestrator import is_open_ended from core.config import SESSION_TTL_SECONDS, MAX_SESSIONS from middleware.auth import AuthUser, get_current_user @@ -52,7 +52,6 @@ logger = logging.getLogger("uvicorn.error") router = APIRouter() -router.include_router(eval_router, prefix="/eval") # --------------------------------------------------------------------------- # ARCH-02: Session context store with TTL eviction and size cap @@ -107,8 +106,6 @@ def _set_session(key: str, data: Dict[str, Any]) -> None: _session_timestamps[key] = _time.monotonic() -import asyncio - # B2 fix: @router.on_event("startup") silently no-ops on APIRouter — the # eviction loop is registered on the FastAPI app instance in main.py instead. # See: _start_session_eviction_loop() in main.py. @@ -125,6 +122,18 @@ class ProcessRequest(BaseModel): session_id: Optional[str] = None +class CreateWorkspaceRequest(BaseModel): + """Request body for the POST /workspaces endpoint. + + P2-05 fix: replaced raw Dict[str, Any] with a typed Pydantic model so + OpenAPI clients get a proper schema and FastAPI returns 422 (not 500) + when required fields are missing. + """ + + name: str + description: Optional[str] = None + + # --------------------------------------------------------------------------- # Upload / Process / Query # --------------------------------------------------------------------------- @@ -158,10 +167,14 @@ async def list_files( workspace_id: Optional[str] = Query(default=None), auth: AuthUser = Depends(get_current_user), ) -> List[Dict[str, Any]]: - """Lists files uploaded by the user, optionally scoped to a workspace.""" + """Lists files uploaded by the current user, optionally scoped to a workspace. + + P2-04 fix: passes user_id so the Supabase query filters by the + authenticated user, preventing cross-user file visibility. + """ try: from services.supabase_service import list_uploaded_files # pylint: disable=import-outside-toplevel - return await list_uploaded_files(workspace_id=workspace_id) + return await list_uploaded_files(user_id=auth.user_id, workspace_id=workspace_id) except Exception as exc: # pylint: disable=broad-except raise HTTPException(status_code=500, detail=str(exc)) from exc @@ -175,6 +188,9 @@ async def process_batch( Merges results into the session's existing context rather than replacing it wholesale, so multiple /process calls accumulate files correctly. + + BUG 4 fix: Includes ``parse_warnings`` in the response so the frontend + can alert the user to near-empty files before they start an agent run. """ session_key = request.session_id or str(auth.user_id) logger.info( @@ -196,16 +212,35 @@ async def process_batch( **existing.get("combined_extractions", {}), **new_details.get("combined_extractions", {}), } + + # BUG 4: accumulate parse_warnings across multiple /process calls + existing_warnings = existing.get("parse_warnings", []) + new_warnings = new_details.get("parse_warnings", []) + merged_warnings = existing_warnings + new_warnings + + # Log each warning so server-side monitoring catches sparse files + for pw in new_warnings: + logger.warning( + "[Process] ParseWarning for '%s': %s", + pw.get("filename", "?"), + pw.get("message", ""), + ) + merged = { **new_details, "combined_extractions": merged_extractions, "files_processed": len(merged_extractions), + "parse_warnings": merged_warnings, } _set_session(session_key, merged) + # Surface parse_warnings in the HTTP response so the React frontend + # can show them immediately (before any agent run is started). + result["parse_warnings"] = new_warnings return result + # --------------------------------------------------------------------------- # Agent run — SSE streaming # --------------------------------------------------------------------------- @@ -240,6 +275,7 @@ async def agent_run( temperature=request.temperature, user_id=user_id, workspace_id=request.workspace_id, + http_request=http_request, # P2-02: enables disconnect monitoring ) else: logger.info( @@ -461,13 +497,17 @@ async def workspace_stats( @router.post("/workspaces") async def create_workspace( - payload: Dict[str, Any], + payload: CreateWorkspaceRequest, auth: AuthUser = Depends(get_current_user), ) -> Dict[str, Any]: - """Creates a new workspace for the authenticated user.""" - name = payload.get("name", "").strip() + """Creates a new workspace for the authenticated user. + + P2-05 fix: accepts a typed Pydantic body instead of raw Dict; field + validation (name required, max length) is handled by the model. + """ + name = payload.name.strip() if not name: - raise HTTPException(status_code=422, detail="Workspace name is required.") + raise HTTPException(status_code=422, detail="Workspace name cannot be blank.") try: from services.supabase_service import create_workspace as _create # pylint: disable=import-outside-toplevel return await _create(user_id=auth.user_id, name=name) @@ -475,6 +515,127 @@ async def create_workspace( raise HTTPException(status_code=500, detail=str(exc)) from exc +@router.post("/workspaces/{workspace_id}/upload") +async def workspace_upload_files( + workspace_id: str, + files: List[UploadFile] = File(...), + session_id: Optional[str] = Query(default=None), + auth: AuthUser = Depends(get_current_user), +) -> Dict[str, Any]: + """Uploads one or more files into a workspace session. + + Saves each file to disk at /workspace/{workspace_id}/{filename}, + extracts schema_json and row_count via the appropriate parser, + inserts a workspace_files row, then returns a MultiFileContext + with all current workspace files and inferred join candidates. + + Args: + workspace_id: Target workspace UUID (from path). + files: Multipart file upload(s). + session_id: Optional session identifier for in-memory cache scoping. + auth: Authenticated user (from JWT). + + Returns: + JSON with accepted_files, rejected_files, and multi_file_context. + """ + effective_session = session_id or str(auth.user_id) + file_names = [f.filename for f in files] + logger.info( + "[WorkspaceUpload] workspace=%s user=%s files=%s (%d)", + workspace_id[:8], + str(auth.user_id)[:8], + ", ".join(file_names), + len(file_names), + ) + try: + return await handle_workspace_upload( + files=files, + workspace_id=workspace_id, + user_id=str(auth.user_id), + session_id=effective_session, + ) + except Exception as exc: # pylint: disable=broad-except + raise HTTPException(status_code=500, detail=str(exc)) from exc + + +@router.delete("/workspaces/{workspace_id}/files/{file_id}") +async def delete_workspace_file( + workspace_id: str, + file_id: str, + auth: AuthUser = Depends(get_current_user), +) -> Dict[str, Any]: + """Removes a single file from a workspace and returns the refreshed MultiFileContext. + + Deletes the workspace_files row (verified to belong to the authenticated user), + removes the physical file from disk (best-effort), then re-runs schema_merger + on the remaining files to produce an updated MultiFileContext. + + Args: + workspace_id: Workspace UUID (from path). + file_id: UUID of the workspace_files row to delete. + auth: Authenticated user (from JWT). + + Returns: + JSON with deleted (bool) and multi_file_context. + """ + logger.info( + "[WorkspaceFileDelete] workspace=%s file=%s user=%s", + workspace_id[:8], + file_id[:8], + str(auth.user_id)[:8], + ) + try: + return await handle_delete_workspace_file( + file_id=file_id, + workspace_id=workspace_id, + user_id=str(auth.user_id), + ) + except ValueError as ve: + raise HTTPException(status_code=404, detail=str(ve)) from ve + except Exception as exc: # pylint: disable=broad-except + raise HTTPException(status_code=500, detail=str(exc)) from exc + + +@router.get("/workspaces/{workspace_id}/files") +async def list_workspace_files_endpoint( + workspace_id: str, + auth: AuthUser = Depends(get_current_user), +) -> List[Dict[str, Any]]: + """Lists files belonging to a workspace, formatted for the frontend upload panel. + + Returns workspace_files rows with keys the frontend expects: + id, filename, file_size, file_url, file_type, schema_json, row_count. + + Args: + workspace_id: Workspace UUID (from path). + auth: Authenticated user (from JWT). + + Returns: + List of file metadata dicts. + """ + try: + from services.supabase_service import list_workspace_files # pylint: disable=import-outside-toplevel + rows = await list_workspace_files( + workspace_id=workspace_id, + user_id=str(auth.user_id), + ) + # Reshape for frontend compatibility + return [ + { + "id": r.get("id"), + "filename": r.get("file_name", ""), + "file_size": r.get("file_size") or 0, + "file_url": r.get("file_url") or "", + "file_type": r.get("file_type", ""), + "schema_json": r.get("schema_json"), + "row_count": r.get("row_count"), + } + for r in rows + ] + except Exception as exc: # pylint: disable=broad-except + raise HTTPException(status_code=500, detail=str(exc)) from exc + + # --------------------------------------------------------------------------- # Cache management # --------------------------------------------------------------------------- diff --git a/backend/core/analyzer/file_analyzer.py b/backend/core/analyzer/file_analyzer.py index f151a1a..2813b4a 100644 --- a/backend/core/analyzer/file_analyzer.py +++ b/backend/core/analyzer/file_analyzer.py @@ -173,6 +173,100 @@ async def analyze(self, combined_extractions: Dict[str, Any], session_id: str = ) return description + async def analyze_multi( + self, + context, # MultiFileContext + session_id: str = "__anon__", + ) -> str: + """Builds a data description from a MultiFileContext. + + For single-file contexts, delegates to the standard ``analyze()`` path + so the existing orchestrator behaviour is preserved exactly. + + For multi-file contexts (2+ files), prepends the compact + ``MultiFileContext.to_prompt_str()`` summary and appends a multi-file + analysis instruction so the LLM knows join keys are available. + + Args: + context: A MultiFileContext instance from schema_merger. + session_id: Session identifier for file content lookup. + + Returns: + Multi-section data description string ready for agent prompts. + """ + from models.multi_file_context import MultiFileContext # pylint: disable=import-outside-toplevel + + files = context.files if context else [] + + if not files: + return "No data files are available in the current context." + + if len(files) == 1: + # Single-file: fall through to the standard path + # Build a minimal combined_extractions dict from the FileSchema + fs = files[0] + dummy_extraction = { + "source_type": fs.file_type, + "sanitized_content": "", + "metadata": { + "columns": [c.name for c in fs.columns], + "dtypes": {c.name: c.dtype for c in fs.columns}, + "row_count": fs.row_count, + "shape": [fs.row_count, len(fs.columns)], + "sample_rows": [ + {c.name: (c.sample_values[0] if c.sample_values else None) for c in fs.columns} + ], + }, + } + return await self.analyze( + {fs.file_name: dummy_extraction}, + session_id=session_id, + ) + + # Multi-file path + self._reset_chain() + + # Build per-file descriptions using LLM or static fallback + file_sections = [] + for fs in files: + dummy_extraction = { + "source_type": fs.file_type, + "sanitized_content": "", + "metadata": { + "columns": [c.name for c in fs.columns], + "dtypes": {c.name: c.dtype for c in fs.columns}, + "row_count": fs.row_count, + "shape": [fs.row_count, len(fs.columns)], + "sample_rows": [ + {c.name: (c.sample_values[0] if c.sample_values else None) for c in fs.columns} + ], + }, + } + section = await self._analyze_file_with_llm(fs.file_name, dummy_extraction, session_id) + if section is None: + section = self._analyze_file_static(fs.file_name, dummy_extraction) + file_sections.append(section) + + # Compose multi-file description with join context header + join_context_header = ( + "=== MULTI-FILE WORKSPACE ===\n" + + context.to_prompt_str() + + "\n\nYou are analysing a multi-file workspace. The user may ask questions " + "that require joining data across files. The detected join candidates are " + "listed above — treat these as probable but not certain. " + "Flag any ambiguity in your analysis output." + ) + + description = join_context_header + "\n\n=== DATA DESCRIPTION ===\n\n" + "\n\n".join(file_sections) + + logger.info( + "[FileAnalyzer] Multi-file description: %d files, %d chars", + len(files), + len(description), + ) + return description + + async def _analyze_file_with_llm( self, filename: str, doc: Dict[str, Any], session_id: str = "__anon__" ) -> "str | None": @@ -213,10 +307,40 @@ async def _analyze_file_with_llm( line for line in lines if not line.strip().startswith("```") ) - # Inject file content into the script preamble + # Inject file content into the script preamble. + # First try the in-memory session cache; fall back to the workspace + # disk path so files uploaded via /workspaces/{id}/upload are found + # even when the session cache is keyed differently. from services.upload_service import get_file_content # pylint: disable=import-outside-toplevel raw_bytes = get_file_content(filename, session_id=session_id) or b"" + if not raw_bytes: + # Disk fallback: scan /workspace/ for the file + workspace_base = os.environ.get("WORKSPACE_FILES_DIR", "/workspace") + for _root, _dirs, _files in os.walk(workspace_base): + if filename in _files: + _candidate = os.path.join(_root, filename) + try: + with open(_candidate, "rb") as _fh: + raw_bytes = _fh.read() + except OSError: + pass + if raw_bytes: + logger.info( + "[FileAnalyzer] Loaded %s from disk fallback (%s).", + filename, _candidate, + ) + break + + if not raw_bytes: + # No data anywhere — skip expensive LLM path, use static fallback + logger.warning( + "[FileAnalyzer] No content found for %s (session=%s); " + "using static fallback.", + filename, session_id, + ) + return None + preamble = ( "import sys, io, json, re\n" "import pandas as _pd\n" diff --git a/backend/core/coder/coder_agent.py b/backend/core/coder/coder_agent.py index b152ace..9b1eec9 100644 --- a/backend/core/coder/coder_agent.py +++ b/backend/core/coder/coder_agent.py @@ -170,20 +170,100 @@ def _is_data_distribution_query(query: str) -> bool: ) -def _build_distribution_fallback_script() -> str: +def _extract_target_columns( + query: str, known_columns: Sequence[str] +) -> List[str]: + """Extracts column names the user explicitly mentioned from their query. + + Cross-references the query text against the known schema columns + (case-insensitive) and returns a list of matched column names in + their original casing. + + This allows fallback scripts to target only the columns the user + asked about instead of plotting everything generically. + + Examples: + >>> _extract_target_columns("show distribution of speed", ["hp", "speed", "attack"]) + ['speed'] + >>> _extract_target_columns("histogram of hp and attack", ["hp", "speed", "attack"]) + ['hp', 'attack'] + >>> _extract_target_columns("explore data", ["hp", "speed", "attack"]) + [] + + Args: + query: The user's natural language query. + known_columns: Column names from the schema hints. + + Returns: + List of matched column names in their original casing. + """ + if not query or not known_columns: + return [] + q_lower = query.lower() + matched: List[str] = [] + for col in known_columns: + # Skip very short column names (1-2 chars) to avoid false positives + # e.g. "hp" matching inside "chart" — but "hp" is a common column + # name so we only skip single-char names. + if len(col) < 2: + continue + if col.lower() in q_lower: + matched.append(col) + return matched + + +def _build_distribution_fallback_script( + target_columns: Optional[List[str]] = None, +) -> str: """Builds a deterministic data-distribution script. Used as an emergency fallback when the LLM endpoint repeatedly fails. Discovers all tabular files in the working directory, prints .describe(), - value counts for categorical columns, and saves interactive Plotly HTML - histograms for numeric ones. + and saves Plotly PNG charts. + + When ``target_columns`` is provided (extracted from the user's query), + the script plots ONLY those specific columns. When empty/None, it falls + back to plotting the first 6 numeric columns generically. - Uses plotly (pre-installed) instead of matplotlib (not available in sandbox). + Uses plotly + kaleido (pre-installed) instead of matplotlib (not available in sandbox). IMPORTANT: This function must produce a syntactically valid Python script. Avoid embedding literal newlines inside f-string quotes — use separate print() calls or string concatenation instead. + + Args: + target_columns: Optional list of specific column names to plot. + When provided, only these columns are visualized. """ + # Build the column-selection logic based on whether we have targets + if target_columns: + # Targeted mode: only plot the user-requested columns + cols_literal = repr(target_columns) + col_selection = ( + f" _target = {cols_literal}\n" + " # Match target columns case-insensitively\n" + " _cols_lower = {c.lower(): c for c in df.columns}\n" + " cols_to_plot = []\n" + " for t in _target:\n" + " actual = _cols_lower.get(t.lower())\n" + " if actual is not None:\n" + " cols_to_plot.append(actual)\n" + " else:\n" + " print('WARNING: column ' + repr(t) + ' not found. Available: ' + str(df.columns.tolist()))\n" + " if not cols_to_plot:\n" + " print('No matching columns found — falling back to first 6 numeric.')\n" + " num_cols = df.select_dtypes(include='number').columns.tolist()\n" + " cols_to_plot = num_cols[:6]\n" + ) + chart_title_expr = "'Distribution of ' + ', '.join(cols_to_plot)" + else: + # Generic mode: plot first 6 numeric columns + col_selection = ( + " num_cols = df.select_dtypes(include='number').columns.tolist()\n" + " cols_to_plot = num_cols[:6]\n" + ) + chart_title_expr = "'Data Distribution: ' + fname" + return ( "import os\n" "import pandas as pd\n" @@ -234,19 +314,18 @@ def _build_distribution_fallback_script() -> str: " print('Descriptive Statistics:')\n" " print(df.describe(include='all').to_string())\n" "\n" - " # ── Plotly: numeric distributions ──\n" + " # ── Plotly: distributions ──\n" " if _HAS_PLOTLY:\n" - " num_cols = df.select_dtypes(include='number').columns.tolist()\n" - " if num_cols:\n" - " n_plots = min(len(num_cols), 6)\n" - " cols_to_plot = num_cols[:n_plots]\n" + + col_selection + + " if cols_to_plot:\n" + " n_plots = len(cols_to_plot)\n" " fig = sp.make_subplots(\n" " rows=n_plots, cols=1,\n" " subplot_titles=['Distribution of ' + c for c in cols_to_plot],\n" " vertical_spacing=0.06,\n" " )\n" " for i, col in enumerate(cols_to_plot, start=1):\n" - " series = df[col].dropna()\n" + " series = pd.to_numeric(df[col], errors='coerce').dropna()\n" " fig.add_trace(\n" " go.Histogram(x=series, nbinsx=30, name=col,\n" " marker_color='#636EFA'),\n" @@ -255,38 +334,16 @@ def _build_distribution_fallback_script() -> str: " fig.update_xaxes(title_text=col, row=i, col=1)\n" " fig.update_yaxes(title_text='Frequency', row=i, col=1)\n" " fig.update_layout(\n" - " title_text='Data Distribution: ' + fname,\n" + f" title_text={chart_title_expr},\n" " height=350 * n_plots,\n" " showlegend=False,\n" " template='plotly_white',\n" " )\n" " base = fname.rsplit('.', 1)[0]\n" - " chart_name = 'distribution_' + base + '.html'\n" - " fig.write_html('./outputs/' + chart_name)\n" - " print('')\n" - " print('Interactive distribution chart saved to ./outputs/' + chart_name)\n" - "\n" - " # ── Plotly: categorical value counts ──\n" - " cat_cols = [c for c in df.columns\n" - " if df[c].dtype == object or str(df[c].dtype) == 'string']\n" - " for col in cat_cols[:4]:\n" - " vc = df[col].value_counts().head(15)\n" - " fig_cat = px.bar(\n" - " x=vc.index.astype(str), y=vc.values,\n" - " labels={'x': col, 'y': 'Count'},\n" - " title='Value Counts: ' + col,\n" - " template='plotly_white',\n" - " )\n" - " cat_chart = 'valuecounts_' + col[:30] + '_' + base + '.html'\n" - " fig_cat.write_html('./outputs/' + cat_chart)\n" - " print('Category chart saved to ./outputs/' + cat_chart)\n" - " else:\n" - " # Plain-text fallback when plotly is also missing\n" - " cat_cols = df.select_dtypes(include='object').columns.tolist()\n" - " for col in cat_cols[:5]:\n" + " chart_name = 'distribution_' + base + '.png'\n" + " fig.write_image('./outputs/' + chart_name)\n" " print('')\n" - " print(\"Value counts for '\" + col + \"':\")\n" - " print(df[col].value_counts().head(10).to_string())\n" + " print('Distribution chart saved to ./outputs/' + chart_name)\n" "\n" " except Exception as exc:\n" " print(str(fname) + ': failed to process (' + str(exc) + ')')\n" @@ -501,21 +558,22 @@ def _extract_code_from_model_text( TASK TYPE OUTPUT MODES: - Insight / Data Analysis: print() final numeric answers clearly. -- Visualization: save interactive charts with fig.write_html('./outputs/.html') - (uses plotly — matplotlib is NOT available). +- Visualization: save charts as PNG images with fig.write_image('./outputs/.png') + (uses plotly + kaleido — matplotlib is NOT available). - Data Wrangling: save cleaned data with df.to_csv('./outputs/.csv', index=False) - Machine Learning: save the model with joblib.dump(model, './outputs/model.joblib') AND print metrics (accuracy, RMSE, etc.) GENERAL RULES: - PRE-INSTALLED PACKAGES (use freely): pandas, numpy, scipy, sklearn, joblib, - plotly, Pillow (PIL), and the Python standard library. + plotly, kaleido, Pillow (PIL), and the Python standard library. - NOT INSTALLED — NEVER import these: matplotlib, seaborn, statsmodels. If you must use visualization, always use plotly (plotly.express or plotly.graph_objects). - Read files by filename — files are pre-injected into the working directory. -- For plots: use plotly.express or plotly.graph_objects. Save with - fig.write_html('./outputs/.html') OR fig.write_image('./outputs/.png') - (write_image requires kaleido — prefer write_html which always works). +- For plots: use plotly.express or plotly.graph_objects. ALWAYS save as PNG: + fig.write_image('./outputs/.png') + Do NOT use fig.write_html() — the UI cannot display HTML artifacts inline. + kaleido is pre-installed so write_image() always works. NEVER call plt.show() or import matplotlib. - The ./outputs/ directory is pre-created. - Handle missing values gracefully with pd.to_numeric(..., errors='coerce'). @@ -548,6 +606,23 @@ def _extract_code_from_model_text( - Coerce numerics: pd.to_numeric(df['col'], errors='coerce') - Drop NaN before stats: df.dropna(subset=['col1', 'col2']) - NEVER silently output NaN as the final result. + +MULTI-FILE MERGE RULES — apply ONLY when the MULTI-FILE CONTEXT block is present: +- The data-loading header (df1 = pd.read_csv(...), df2 = ...) is ALREADY + provided at the top of the script. DO NOT re-load the files. +- When a merge is required, use EXPLICIT left_on/right_on parameters: + merged = pd.merge(df1, df2, left_on='', right_on='', how='inner') + Never use 'on=' alone when the column names differ between files. +- After every pd.merge(), immediately print the resulting shape: + print(f'merged shape: {{merged.shape}}') + If merged.shape[0] == 0, print a warning and stop gracefully. +- Prefer the highest-confidence join candidate from the MULTI-FILE CONTEXT + unless the user's query explicitly names different columns. +- Validate that the join columns exist in BOTH DataFrames before merging: + for _df, _col, _name in [(df1, '', 'df1'), (df2, '', 'df2')]: + if _col not in _df.columns: + print(f'ERROR: {{_col!r}} not in {{_name}}: {{_df.columns.tolist()}}') + raise KeyError(_col) """ _CODER_HUMAN = """\ @@ -740,6 +815,7 @@ async def generate_code( execution_output: str = "", schema_hints: str = "", token_tracker: Optional[TokenTracker] = None, + multi_file_context=None, # Optional[MultiFileContext] ) -> str: """Generates a Python script implementing the analysis plan. @@ -765,12 +841,30 @@ async def generate_code( # Cap execution output passed to coder to avoid context window exhaustion trimmed_exec = execution_output[:_MAX_EXEC_OUTPUT_CHARS] if execution_output else "(none)" + # Multi-file: prepend the deterministic data-loading header to previous_code + # so the coder extends it rather than writing its own pd.read_* calls. + effective_previous_code = previous_code + if multi_file_context is not None and len(multi_file_context.files) >= 2: + if not previous_code or previous_code.strip() in ("", "(none — this is round 1, write from scratch)"): + # Round 1: inject the reader header as the starting script + reader_header = multi_file_context.to_reader_header( + workspace_id="" + ) + effective_previous_code = reader_header + # Also extend data_description with join context + join_block = ( + "\n\nMULTI-FILE CONTEXT:\n" + + multi_file_context.to_prompt_str() + + "\n" + ) + data_description = data_description + join_block + invoke_input = { "query": query, "data_description": data_description, "schema_hints": schema_hints or "(none provided)", "plan_steps": formatted_steps, - "previous_code": previous_code or "(none — this is round 1, write from scratch)", + "previous_code": effective_previous_code or "(none — this is round 1, write from scratch)", "execution_output": trimmed_exec, } @@ -860,11 +954,15 @@ async def generate_code( return _build_column_listing_fallback_script(known_columns).strip() if _is_data_distribution_query(query): + _target_cols = _extract_target_columns(query, known_columns) logger.warning( "[Coder] Raw completion failed twice for distribution query; " - "using deterministic fallback script." + "using deterministic fallback script (target_cols=%s).", + _target_cols or "all", ) - return _build_distribution_fallback_script().strip() + return _build_distribution_fallback_script( + target_columns=_target_cols or None + ).strip() raise ValueError( "[Coder] Raw completion failed (primary + fresh fallback). " @@ -961,11 +1059,15 @@ async def generate_code( ) return _build_column_listing_fallback_script(known_columns).strip() if _is_data_distribution_query(query): + _target_cols = _extract_target_columns(query, known_columns) logger.warning( "[Coder] Structured+raw fallback failed for distribution query; " - "using deterministic fallback script." + "using deterministic fallback script (target_cols=%s).", + _target_cols or "all", ) - return _build_distribution_fallback_script().strip() + return _build_distribution_fallback_script( + target_columns=_target_cols or None + ).strip() raise ValueError( f"[Coder] Structured output + raw fallback both failed. " f"Structured error: {exc_payload[:150]}, " diff --git a/backend/core/config.py b/backend/core/config.py index 734cb96..d1613f6 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -45,9 +45,6 @@ # The previous restriction to csv/xlsx/json prevented hard-task benchmarks. ANALYSIS_MODE_ALLOWED_FORMATS: Set[str] = set(ALLOWED_MIME_TYPES.keys()) -# IDP mode accepts the same full set -IDP_ALLOWED_FORMATS: Set[str] = set(ALLOWED_MIME_TYPES.keys()) - # --------------------------------------------------------------------------- # Supabase credentials # --------------------------------------------------------------------------- @@ -129,6 +126,34 @@ DOCKER_MEMORY_LIMIT: str = os.getenv("DOCKER_MEMORY_LIMIT", "512m") DOCKER_CPU_QUOTA: float = float(os.getenv("DOCKER_CPU_QUOTA", "0.5")) +# --------------------------------------------------------------------------- +# Subprocess sandbox resource limits (BUG 1 fix) +# --------------------------------------------------------------------------- +# Applied via RLIMIT_CPU / RLIMIT_AS inside the forked child on Unix. +# Keeps a runaway / malicious script from exhausting host CPU or RAM. + +# Max CPU seconds the sandbox script may consume before SIGXCPU / SIGKILL. +SANDBOX_CPU_TIME_LIMIT_SECONDS: int = int( + os.getenv("SANDBOX_CPU_TIME_LIMIT_SECONDS", "30") +) + +# Max virtual address space bytes (512 MB default) — prevents fork-bombs and +# memory-hungry numpy/pandas operations from OOM-killing the host process. +SANDBOX_MEMORY_LIMIT_BYTES: int = int( + os.getenv("SANDBOX_MEMORY_LIMIT_BYTES", str(512 * 1024 * 1024)) +) + +# --------------------------------------------------------------------------- +# Token budget reserve fraction (BUG 3 fix) +# --------------------------------------------------------------------------- +# Fraction of MAX_TOKENS_PER_RUN that must remain *after* context injection +# for the agent stages (Planner + Coder + Verifier). If context consumes +# more than (1 - reserve), the schema is truncated and a warning is emitted. + +CONTEXT_BUDGET_RESERVE_FRACTION: float = float( + os.getenv("CONTEXT_BUDGET_RESERVE_FRACTION", "0.20") +) + # --------------------------------------------------------------------------- # Session context management (ARCH-02) # --------------------------------------------------------------------------- diff --git a/backend/core/debugger/debugger_agent.py b/backend/core/debugger/debugger_agent.py index 7586c1b..a2f8e71 100644 --- a/backend/core/debugger/debugger_agent.py +++ b/backend/core/debugger/debugger_agent.py @@ -86,10 +86,16 @@ class DebuggerOutput(BaseModel): - Missing import → add it at the top of the script. - KeyError on column → use df.get(col) or check df.columns first. - Division by zero / NaN result → add a zero-variance guard. -4. After fixing, return the ENTIRE corrected script (not just the diff). -5. Output ONLY raw Python. No markdown fences (``` or `python`). -6. Classify the error_type from the traceback accurately. -7. Write a concise fix_summary (one sentence maximum). +4. MULTI-FILE MERGE ERRORS — when a MergeError, KeyError on a join key, or + empty-merge result is detected AND a MULTI-FILE CONTEXT block is provided: + - Switch to explicit left_on/right_on parameters if 'on=' was used. + - Verify that the join columns exist in both DataFrames before merging. + - If the merge produces 0 rows, try switching how='left' and log a warning. + - Check for casing mismatches (e.g. 'ID' vs 'id') using str.lower(). +5. After fixing, return the ENTIRE corrected script (not just the diff). +6. Output ONLY raw Python. No markdown fences (``` or `python`). +7. Classify the error_type from the traceback accurately. +8. Write a concise fix_summary (one sentence maximum). """ _DEBUGGER_HUMAN = """\ @@ -167,6 +173,7 @@ async def debug( plan_steps: List[Dict[str, Any]], schema_context: str = "", token_tracker: Optional[TokenTracker] = None, + multi_file_context=None, # Optional[MultiFileContext] ) -> Dict[str, Any]: """Repairs the failing script and returns the corrected version. @@ -176,9 +183,10 @@ async def debug( plan_steps: Current plan steps, for context. schema_context: Data schema summary (column names / types) to help the debugger understand data-specific errors. - token_tracker: Optional run-level tracker. When provided, token - usage from this LLM call is recorded automatically via a - LangChain callback. + token_tracker: Optional run-level tracker. + multi_file_context: Optional MultiFileContext. When provided and + a merge-related error is detected, join context is appended + to schema_context so the debugger can fix join key issues. Returns: Dict with keys: @@ -195,6 +203,28 @@ async def debug( trimmed_tb = traceback[-_MAX_TRACEBACK_CHARS:] trimmed_code = code[-_MAX_CODE_CHARS:] + # Detect merge-related errors and append join context when available + _MERGE_ERROR_KEYWORDS = ( + "mergeerror", "merge error", "keyerror", + "mergekey", "left_on", "right_on", "pd.merge", + ) + _tb_lower = (traceback or "").lower() + _is_merge_error = any(kw in _tb_lower for kw in _MERGE_ERROR_KEYWORDS) or ( + ("valueerror" in _tb_lower or "key" in _tb_lower) + and any(kw in _tb_lower for kw in ("merge", "join", "column")) + ) + effective_schema_context = schema_context + if _is_merge_error and multi_file_context is not None and multi_file_context.files: + join_block = ( + "\n\nMULTI-FILE CONTEXT (use this to fix join key errors):\n" + + multi_file_context.to_prompt_str() + + "\n\nIMPORTANT: Use explicit left_on/right_on, never 'on=' when column " + "names differ. Check for case mismatches (e.g. 'ID' vs 'id')." + ) + effective_schema_context = (schema_context or "") + join_block + logger.info( + "[Debugger] Merge error detected — injected MultiFileContext into schema_context." + ) if self._chain is None: built = self._build_chain() # heavy work — no lock held async with self._lock: @@ -208,7 +238,7 @@ async def debug( chain.ainvoke( { "traceback": trimmed_tb or "(no traceback — check stderr)", - "schema_context": schema_context or "(no schema context available)", + "schema_context": effective_schema_context or "(no schema context available)", "code": trimmed_code, "plan_steps": formatted_steps, }, diff --git a/backend/core/deep_research_orchestrator.py b/backend/core/deep_research_orchestrator.py index 834c104..e1b7d3c 100644 --- a/backend/core/deep_research_orchestrator.py +++ b/backend/core/deep_research_orchestrator.py @@ -30,6 +30,35 @@ logger = logging.getLogger("uvicorn.info") +def _make_ds_star_orchestrator( + model: Optional[str], + coder_model: Optional[str], + temperature: Optional[float], +) -> DsStarOrchestrator: + """Instantiates a **fresh** DsStarOrchestrator for a single sub-question run. + + P0 fix: the previous module-level cache (_ds_star_cache) shared one + DsStarOrchestrator instance across all concurrent sub-question tasks. + Because DsStarOrchestrator.coder (CoderAgent) carries mutable run-level + state (``_raw_mode_engaged``), a single schema failure in any parallel + task would call ``force_raw_completion()`` and permanently corrupt the + shared instance for ALL concurrent and future sub-questions. + + Instantiating a fresh orchestrator per sub-question is the only safe + option: each object's agents are stateless between instantiation and the + end of ``run()``, so the 8-agent construction cost (~1 ms, no I/O) is + negligible compared to the LLM round-trip latency. + """ + logger.debug( + "[DeepResearch] Creating fresh DsStarOrchestrator — model=%s, coder=%s", + model, coder_model, + ) + return DsStarOrchestrator( + model=model, + coder_model=coder_model, + temperature=temperature, + ) + # --------------------------------------------------------------------------- # Open-ended query classifier @@ -70,21 +99,21 @@ def is_open_ended(query: str) -> bool: async def _run_single_ds_star( question: str, context: Dict[str, Any], - model: Optional[str], - coder_model: Optional[str], - temperature: Optional[float], + orchestrator: DsStarOrchestrator, max_rounds: int, sub_run_id: str, session_id: str = "__anon__", ) -> Dict[str, Any]: """Runs a complete DS-STAR loop for one sub-question and returns its result. + P1-06 fix: accepts a pre-built, cached ``DsStarOrchestrator`` instead of + instantiating one per sub-question. ``max_rounds`` is forwarded into + ``orchestrator.run()`` (not mutated on the shared instance). + Args: question: The atomic sub-question to answer. context: Processing context passed from the research endpoint. - model: LLM model override (Pro tier). - coder_model: LLM model override for code generation. - temperature: Sampling temperature override. + orchestrator: Shared cached DsStarOrchestrator instance. max_rounds: Maximum orchestrator rounds per sub-question. sub_run_id: Unique run ID for this sub-question run. session_id: Client session identifier — scopes executor file access so @@ -93,13 +122,6 @@ async def _run_single_ds_star( Returns: Dict containing status, execution_output, insights, code, rounds, run_id. """ - orchestrator = DsStarOrchestrator( - max_rounds=max_rounds, - model=model, - coder_model=coder_model, - temperature=temperature, - ) - result: Dict[str, Any] = { "status": "failed", "execution_output": "", @@ -111,7 +133,11 @@ async def _run_single_ds_star( try: async for event in orchestrator.run( - question, context, run_id=sub_run_id, session_id=session_id + question, + context, + run_id=sub_run_id, + session_id=session_id, + max_rounds=max_rounds, ): event_type = event.get("event") payload = event.get("payload", {}) @@ -199,6 +225,7 @@ async def run( context: Dict[str, Any], report_id: str = "", session_id: str = "__anon__", + max_rounds: Optional[int] = None, ) -> AsyncGenerator[Dict[str, Any], None]: """Executes the DS-STAR+ research loop and yields SSE events. @@ -210,10 +237,17 @@ async def run( session_id: Client session identifier — forwarded to all sub-question DS-STAR runs so their executors access the correct session file bucket instead of ``__anon__``. + max_rounds: Per-call round limit for sub-question runs. + When provided, overrides ``self._max_rounds`` for this + invocation only (P1-01 fix: no mutation of shared state). Yields: AgentEvent dicts for SSE streaming. """ + # Resolve effective rounds locally — never write to self._max_rounds + _effective_max_rounds: int = ( + max_rounds if max_rounds is not None else self._max_rounds + ) run_t0 = time.monotonic() report_id = report_id or uuid.uuid4().hex combined = context.get("combined_extractions", {}) @@ -296,24 +330,33 @@ async def run( semaphore = asyncio.Semaphore(self._max_workers) async def _run_with_semaphore(i: int, question: str) -> Dict[str, Any]: + # Build the started event BEFORE acquiring the semaphore so + # its data is ready, but it is returned alongside the result + # for the outer loop to yield in order. + started_event = _event( + "subquestion_started", + message=f"[Q{i + 1}] Running: {question[:80]}", + index=i, + question=question, + sub_run_id=sub_run_ids[i], + ) async with semaphore: - yield_event = _event( - "subquestion_started", - message=f"[Q{i + 1}] Running: {question[:80]}", - index=i, - question=question, - sub_run_id=sub_run_ids[i], - ) + # P1-03 fix: event is now created BEFORE the await so it + # accurately represents when the task entered execution. logger.info( "[DeepResearch] Q%d started | run_id=%s", i + 1, sub_run_ids[i] ) + # P0 fix: always construct a fresh orchestrator per sub-question. + # Sharing a single instance causes CoderAgent._raw_mode_engaged + # to leak across parallel tasks on schema failure. + sub_orchestrator = _make_ds_star_orchestrator( + self._model, self._coder_model, self._temperature + ) result = await _run_single_ds_star( question=question, context=context, - model=self._model, - coder_model=self._coder_model, - temperature=self._temperature, - max_rounds=self._max_rounds, + orchestrator=sub_orchestrator, + max_rounds=_effective_max_rounds, sub_run_id=sub_run_ids[i], session_id=session_id, ) @@ -323,7 +366,7 @@ async def _run_with_semaphore(i: int, question: str) -> Dict[str, Any]: result["status"], sub_run_ids[i], ) - return yield_event, result + return started_event, result # Run all sub-questions concurrently under semaphore tasks = [ @@ -422,13 +465,16 @@ async def _run_with_semaphore(i: int, question: str) -> Dict[str, Any]: async def _sup_run(i: int, question: str) -> Dict[str, Any]: async with sup_semaphore: + # P0 fix: fresh orchestrator per supplementary question — + # same race-condition isolation as the primary run above. + sup_orchestrator = _make_ds_star_orchestrator( + self._model, self._coder_model, self._temperature + ) return await _run_single_ds_star( question=question, context=context, - model=self._model, - coder_model=self._coder_model, - temperature=self._temperature, - max_rounds=self._max_rounds, + orchestrator=sup_orchestrator, + max_rounds=_effective_max_rounds, sub_run_id=sup_run_ids[i], session_id=session_id, ) diff --git a/backend/core/ds_star_orchestrator.py b/backend/core/ds_star_orchestrator.py index 44781bf..9d69e63 100644 --- a/backend/core/ds_star_orchestrator.py +++ b/backend/core/ds_star_orchestrator.py @@ -7,7 +7,6 @@ Gap fixes applied: - Lambda capture bug fixed: coro_factory args now captured at call-site via functools.partial / default-argument binding instead of closure. -- run_id correctly passed into RunMetrics (was always empty string). - Complexity classification now uses a heuristic (file count + query keywords) rather than a pure file-count label. - REMOVE_STEPS router action now wired to PlannerAgent.remove_steps_from(). @@ -39,9 +38,8 @@ from core.planner.planner_agent import PlannerAgent from core.router.router_agent import RouterAgent from core.verifier.verifier_agent import VerifierAgent -from core.config import MAX_AGENT_ROUNDS, MAX_DEBUGGER_RETRIES, MAX_TOKENS_PER_RUN +from core.config import MAX_AGENT_ROUNDS, MAX_DEBUGGER_RETRIES, MAX_TOKENS_PER_RUN, CONTEXT_BUDGET_RESERVE_FRACTION from core.token_tracker import TokenTracker -from models.metrics_schema import RoundMetric, RoundTimingCollector, RunMetrics logger = logging.getLogger("uvicorn.info") @@ -244,6 +242,8 @@ async def run( context: Dict[str, Any], run_id: str = "", session_id: str = "__anon__", + max_rounds: Optional[int] = None, + workspace_id: Optional[str] = None, ) -> AsyncGenerator[Dict[str, Any], None]: """Executes the DS-STAR loop and yields SSE events. @@ -254,15 +254,22 @@ async def run( run_id: Unique run identifier for metrics (passed from controller). session_id: Client session identifier — used to scope file cache access so the executor only sees this session's uploaded files. + max_rounds: Per-call round limit. When provided, overrides + ``self._max_rounds`` for this invocation only so concurrent + requests with different limits share the cached instance safely + (P1-01 fix: no mutation of shared state). + workspace_id: Optional workspace UUID. Forwarded to the executor + so it can load files from disk when the session cache is empty. Yields: AgentEvent dicts for SSE streaming. """ + # Resolve effective max_rounds locally — never write to self._max_rounds + _effective_max_rounds: int = max_rounds if max_rounds is not None else self._max_rounds run_t0 = time.monotonic() execution_logs: List[str] = [] combined = context.get("combined_extractions", {}) pending_retry_events: List[Dict] = [] - round_metrics: List[RoundMetric] = [] file_count = len(combined) complexity = _classify_complexity(file_count, query) @@ -289,6 +296,39 @@ async def run( f"[FileAnalyzer] {len(data_description)} chars of data description generated." ) + # ── BUG 3 fix: Context token-budget guard ──────────────────────────── + # Estimate token consumption of the schema context using the standard + # ~4 chars-per-token heuristic. If injecting the full context would + # leave less than CONTEXT_BUDGET_RESERVE_FRACTION of the total budget + # for downstream agent stages (Planner, Coder, Verifier, etc.) we + # truncate data_description to the max char count that fits within the + # safe window and surface a user-facing SSE warning instead of silently + # proceeding and then hitting the hard cap mid-run. + _CHARS_PER_TOKEN = 4 # conservative heuristic for mixed code/text + _total_budget = MAX_TOKENS_PER_RUN + _reserved_tokens = int(_total_budget * CONTEXT_BUDGET_RESERVE_FRACTION) + _context_token_estimate = len(data_description) // _CHARS_PER_TOKEN + _safe_context_tokens = _total_budget - _reserved_tokens + + if _context_token_estimate > _safe_context_tokens: + _max_chars = _safe_context_tokens * _CHARS_PER_TOKEN + data_description = data_description[:_max_chars] + _trunc_msg = ( + f"Schema context was too large ({_context_token_estimate:,} estimated tokens). " + f"Truncated to {_safe_context_tokens:,} tokens to preserve a " + f"{int(CONTEXT_BUDGET_RESERVE_FRACTION * 100)}% budget reserve " + f"({_reserved_tokens:,} tokens) for agent stages." + ) + execution_logs.append(f"[BudgetGuard] {_trunc_msg}") + logger.warning("[Orchestrator] %s", _trunc_msg) + yield _event( + "context_truncated", + message=_trunc_msg, + original_tokens=_context_token_estimate, + truncated_to_tokens=_safe_context_tokens, + reserved_tokens=_reserved_tokens, + ) + # ── Stage 2: Initial Plan ───────────────────────────────────────────── yield _event("planning", message="Creating initial analysis plan…") planner_t0 = time.monotonic() @@ -340,11 +380,8 @@ async def run( consecutive_coder_schema_errors = 0 coder_raw_mode_engaged = False # idempotent guard for force_raw_completion() - for round_num in range(1, self._max_rounds + 1): + for round_num in range(1, _effective_max_rounds + 1): rounds_completed = round_num - timing = RoundTimingCollector(round_num=round_num) - if round_num == 1: - timing.record("planner", planner_ms + analyzer_ms) # ── Token budget check (Gap 4) ───────────────────────────────── if token_tracker.over_budget(): @@ -359,9 +396,9 @@ async def run( yield _event( "round_start", - message=f"Round {round_num}/{self._max_rounds}", + message=f"Round {round_num}/{_effective_max_rounds}", round=round_num, - max_rounds=self._max_rounds, + max_rounds=_effective_max_rounds, ) # ── 3a. Code Generation ─────────────────────────────────────────── @@ -400,7 +437,6 @@ async def run( yield ev pending_retry_events.clear() consecutive_coder_schema_errors = 0 - timing.record("coder", _ms_since(coder_t0)) yield _event( "code_ready", message="Code generated.", @@ -411,7 +447,6 @@ async def run( for ev in pending_retry_events: yield ev pending_retry_events.clear() - timing.record("coder", _ms_since(coder_t0)) exc_str = str(exc) execution_logs.append( f"[Round {round_num}] Coder exhausted retries: {exc_str}" @@ -527,10 +562,11 @@ async def run( _iter_t0 = time.monotonic() # ARCH-06: time each attempt individually try: last_exec_result = await self.executor.run( - _code_to_run, session_id=session_id + _code_to_run, + session_id=session_id, + workspace_id=workspace_id, ) _total_exec_ms += _ms_since(_iter_t0) - timing.record("executor", _total_exec_ms) exec_summary = ( f"[Round {round_num}] Execution " f"{'succeeded' if last_exec_result.success else 'failed'}." @@ -548,7 +584,6 @@ async def run( stderr=last_exec_result.stderr[:500], success=last_exec_result.success, round=round_num, - executor_ms=timing.get("executor"), ) # Emit artifact events @@ -618,7 +653,6 @@ async def run( except Exception as exc: # pylint: disable=broad-except _total_exec_ms += _ms_since(_iter_t0) - timing.record("executor", _total_exec_ms) last_exec_result = ExecutionResult("", str(exc), 1) execution_logs.append(f"[Round {round_num}] Executor crash: {exc}") yield _event("warning", message=f"Executor crash: {exc}") @@ -658,7 +692,6 @@ async def run( for ev in pending_retry_events: yield ev pending_retry_events.clear() - timing.record("verifier", _ms_since(verifier_t0)) execution_logs.append( f"[Round {round_num}] Verifier: sufficient={verification['is_sufficient']}, " f"reason={verification['reason'][:80]}" @@ -673,13 +706,11 @@ async def run( reason=verification["reason"], confidence=verification.get("confidence", 0.5), round=round_num, - verifier_ms=timing.get("verifier"), ) except Exception as exc: # pylint: disable=broad-except for ev in pending_retry_events: yield ev pending_retry_events.clear() - timing.record("verifier", _ms_since(verifier_t0)) verification = { "is_sufficient": False, "reason": str(exc), @@ -690,12 +721,6 @@ async def run( ) yield _event("warning", message=f"Verifier error: {exc}") - round_metrics.append(timing.build( - is_sufficient=verification["is_sufficient"], - verifier_confidence=verification.get("confidence", 0.0), - exec_success=last_exec_result.success, - )) - if verification["is_sufficient"]: rounds_until_sufficient = round_num break # Done ✓ @@ -730,7 +755,6 @@ async def run( for ev in pending_retry_events: yield ev pending_retry_events.clear() - timing.record("router", _ms_since(router_t0)) action = decision.get("action", "ADD_STEP") step_index = decision.get("step_index") @@ -804,13 +828,11 @@ async def run( steps=plan_steps, action=action, round=round_num, - router_ms=timing.get("router"), ) except Exception as exc: # pylint: disable=broad-except for ev in pending_retry_events: yield ev pending_retry_events.clear() - timing.record("router", _ms_since(router_t0)) execution_logs.append( f"[Round {round_num}] Router error: {exc}" ) @@ -818,21 +840,6 @@ async def run( # ── Stage 4: Evaluation Metrics ─────────────────────────────────────── total_run_ms = _ms_since(run_t0) - final_status = ( - "completed" if verification.get("is_sufficient") else "max_rounds_reached" - ) - run_metrics = RunMetrics( - run_id=run_id, # FIX: was always "" — now passed from controller - query=query, - complexity=complexity, - file_count=file_count, - rounds_completed=rounds_completed, - rounds_until_sufficient=rounds_until_sufficient or rounds_completed, - total_run_ms=total_run_ms, - final_status=final_status, - rounds=round_metrics, - ) - # ── Stage 5: Finalize ───────────────────────────────────────────────── artifact_names_final = list(last_exec_result.artifacts.keys()) @@ -886,14 +893,6 @@ async def run( **final_result, ) - yield _event( - "metrics", - message="Run evaluation metrics captured.", - metrics=run_metrics.summary(), - total_run_ms=total_run_ms, - complexity=complexity, - ) - logger.info( "[Orchestrator] Completed — run_id=%s rounds=%d, steps=%d, " "code=%d chars, complexity=%s, total_ms=%d", diff --git a/backend/core/executor/code_executor.py b/backend/core/executor/code_executor.py index b2424fd..624d024 100644 --- a/backend/core/executor/code_executor.py +++ b/backend/core/executor/code_executor.py @@ -12,18 +12,27 @@ - Optional Docker sandbox path controlled by DOCKER_SANDBOX_ENABLED env flag (Gap 2 fix). FAIL-CLOSED: raises RuntimeError when Docker is enabled but unavailable — no subprocess fallback is permitted in production. +- BUG 1 (HIGH) fix: _run_popen_capped() now runs with: + * start_new_session=True so the entire child process group can be killed + atomically on timeout via os.killpg, preventing orphaned processes. + * _apply_resource_limits() as a preexec_fn that sets RLIMIT_CPU and + RLIMIT_AS (Unix only) to prevent runaway CPU/memory consumption. + * SandboxViolationError raised for any constraint breach, keeping raw + OS/subprocess details away from the LLM error context. Artifact MIME types extended to cover image formats and ML model files. """ import asyncio +import threading import base64 import logging import os +import signal import subprocess import sys import tempfile -from typing import Any, Dict +from typing import Any, Dict, Optional from core.config import ( DOCKER_CPU_QUOTA, @@ -31,12 +40,231 @@ DOCKER_SANDBOX_ENABLED, DOCKER_SANDBOX_IMAGE, EXECUTION_TIMEOUT_SECONDS, + SANDBOX_CPU_TIME_LIMIT_SECONDS, + SANDBOX_MEMORY_LIMIT_BYTES, ) logger = logging.getLogger("uvicorn.info") _ANON_SESSION = "__anon__" + +# --------------------------------------------------------------------------- +# Custom exception for sandbox constraint violations (BUG 1 fix) +# --------------------------------------------------------------------------- + +class SandboxViolationError(RuntimeError): + """Raised when a sandboxed script breaches a resource or path constraint. + + Keeping violations as a typed exception allows the orchestrator to handle + them differently from ordinary execution errors (e.g. not retrying on RCE). + """ + + +# --------------------------------------------------------------------------- +# Resource-limit preexec helper (BUG 1 fix — Unix only) +# --------------------------------------------------------------------------- + +def _apply_resource_limits() -> None: # pragma: no cover + """Called by the child process before exec() to install resource caps. + + This function runs *inside the forked child* on Unix systems. It sets: + - RLIMIT_CPU: max CPU time in seconds (soft=limit, hard=limit+5) + - RLIMIT_AS: max virtual address space (soft=limit, hard=limit) + + On Windows ``resource`` is not available; we log a warning and continue + so that the rest of the sandbox machinery (timeout kill, env sanitisation) + still works. + """ + try: + import resource # pylint: disable=import-outside-toplevel + cpu_soft = SANDBOX_CPU_TIME_LIMIT_SECONDS + cpu_hard = cpu_soft + 5 # kernel sends SIGXCPU at soft, SIGKILL at hard + resource.setrlimit(resource.RLIMIT_CPU, (cpu_soft, cpu_hard)) + + mem = SANDBOX_MEMORY_LIMIT_BYTES + resource.setrlimit(resource.RLIMIT_AS, (mem, mem)) + except ImportError: + # Windows — resource limits via preexec_fn are not supported + pass + except Exception as exc: # pylint: disable=broad-except + # Swallow here — we are inside the child; a raised exception would + # cause confusing spawn errors instead of a clean timeout kill. + import sys as _sys # pylint: disable=import-outside-toplevel + print(f"[Sandbox] WARNING: resource limits not applied: {exc}", file=_sys.stderr) + + +# --------------------------------------------------------------------------- +# Output size cap (P1 OOM fix) +# --------------------------------------------------------------------------- + +# Hard limit on captured stdout + stderr per execution. A script that prints +# in an infinite loop (e.g. `while True: print("A" * 1024)`) would otherwise +# exhaust the host OS memory before the timeout fires, crashing the FastAPI +# thread pool. 2 MB per stream is generous for legitimate analytics output. +_MAX_OUTPUT_BYTES: int = 2 * 1024 * 1024 # 2 MB +_OUTPUT_TRUNCATED_SENTINEL = ( + "\n\n[TRUNCATED] Output exceeded 2 MB limit and was cut off." +) + + +def _read_capped(stream, max_bytes: int = _MAX_OUTPUT_BYTES) -> str: + """Reads up to *max_bytes* from a binary stream and decodes to str. + + Designed to be called from a background thread concurrently with the other + stream so neither pipe blocks the process (pipe-deadlock prevention). Any + data beyond the cap is silently drained so the child doesn't stall on a + full pipe buffer. A sentinel string is appended when truncation occurs so + the LLM knows the output ended early rather than cleanly. + + Args: + stream: A readable binary file-like object (subprocess PIPE). + max_bytes: Maximum bytes to read before truncating. + + Returns: + Decoded string, with a truncation notice when the cap was reached. + """ + chunks: list = [] + total = 0 + truncated = False + try: + while True: + chunk = stream.read(65536) # 64 KB reads + if not chunk: + break + remaining = max_bytes - total + if remaining <= 0: + truncated = True + # Keep draining so the child doesn't block on a full pipe + continue + if len(chunk) > remaining: + chunks.append(chunk[:remaining]) + total += remaining + truncated = True + else: + chunks.append(chunk) + total += len(chunk) + except Exception: # pylint: disable=broad-except + pass + text = b"".join(chunks).decode("utf-8", errors="replace") + if truncated: + text += _OUTPUT_TRUNCATED_SENTINEL + logger.warning( + "[Executor] Output truncated at %d bytes — possible infinite print loop.", + max_bytes, + ) + return text + + +def _run_popen_capped( + args: list, + cwd: str, + env: dict, + timeout: int, + allowed_cwd: str = "", +) -> "tuple[str, str, int]": + """Runs a subprocess with concurrent capped pipe readers (deadlock-safe). + + BUG 1 (HIGH) fix — additional hardening over the original: + * ``start_new_session=True`` puts the child in its own process group so + that ``os.killpg`` can reliably kill the *entire* process tree on + timeout, not just the top-level PID. + * ``preexec_fn=_apply_resource_limits`` installs RLIMIT_CPU / RLIMIT_AS + inside the forked child before exec (Unix only). + * Path whitelist check: if ``allowed_cwd`` is provided, any attempt by + the args list to reference a path outside it raises + ``SandboxViolationError`` before the process is even spawned. + * Timeout kill uses ``os.killpg`` (SIGKILL to the whole group) then + ``proc.wait()`` to reap zombies; raw exceptions are wrapped in + ``SandboxViolationError`` to prevent leaking OS internals to callers. + + Args: + args: Command + arguments list for subprocess. + cwd: Working directory for the subprocess. + env: Environment variables dict. + timeout: Wall-clock timeout in seconds. + allowed_cwd: Optional absolute path prefix; any arg that is an + absolute path outside this prefix raises SandboxViolationError. + + Returns: + Tuple of (stdout_text, stderr_text, returncode). + + Raises: + SandboxViolationError: On timeout, resource breach, or path violation. + """ + # ── Path whitelist check (before spawn) ────────────────────────────────── + if allowed_cwd: + _allowed = os.path.realpath(allowed_cwd) + for i, arg in enumerate(args): + # Exempt the executable itself (args[0]) from the sandbox constraint + if i == 0: + continue + if isinstance(arg, str) and os.path.isabs(arg): + if not os.path.realpath(arg).startswith(_allowed): + raise SandboxViolationError( + f"Path constraint violated: '{arg}' is outside sandbox '{_allowed}'. " + "Execution aborted." + ) + + # ── Spawn child in its own session (process group) ─────────────────────── + popen_kwargs: Dict[str, Any] = dict( + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=cwd, + env=env, + ) + _is_unix = hasattr(os, "killpg") + if _is_unix: + # start_new_session=True creates a new process group leader so we + # can send SIGKILL to every child spawned by the script. + popen_kwargs["start_new_session"] = True + popen_kwargs["preexec_fn"] = _apply_resource_limits + + try: + proc = subprocess.Popen(args, **popen_kwargs) + except OSError as exc: + raise SandboxViolationError( + f"Failed to spawn sandbox subprocess: {exc}" + ) from exc + + stdout_holder: list = [""] + stderr_holder: list = [""] + + t_out = threading.Thread( + target=lambda: stdout_holder.__setitem__(0, _read_capped(proc.stdout)), + daemon=True, + ) + t_err = threading.Thread( + target=lambda: stderr_holder.__setitem__(0, _read_capped(proc.stderr)), + daemon=True, + ) + t_out.start() + t_err.start() + + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + # Kill entire process group (Unix) or just the process (Windows) + if _is_unix: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except ProcessLookupError: + pass # Process already exited — nothing to kill + else: + proc.kill() + proc.wait() # reap zombie + t_out.join(timeout=5) + t_err.join(timeout=5) + raise SandboxViolationError( + f"Execution timed out after {timeout} seconds. " + "The process tree was killed." + ) + + t_out.join() + t_err.join() + return stdout_holder[0], stderr_holder[0], proc.returncode + + # --------------------------------------------------------------------------- # Minimal allowed environment variables (credential-safe) # --------------------------------------------------------------------------- @@ -71,6 +299,7 @@ def _safe_env() -> Dict[str, str]: ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".svg": "image/svg+xml", + ".html": "text/html", ".pdf": "application/pdf", ".csv": "text/csv", ".txt": "text/plain", @@ -156,6 +385,7 @@ async def run( self, code: str, session_id: str = _ANON_SESSION, + workspace_id: Optional[str] = None, ) -> ExecutionResult: """Writes code to a temp file and executes it asynchronously. @@ -163,6 +393,9 @@ async def run( code: Python source code to execute. session_id: Session identifier used to fetch only that session's uploaded files into the sandbox (Gap 1 isolation fix). + workspace_id: Optional workspace UUID. When the in-memory session + cache is empty, files are loaded from the workspace disk + directory as a fallback (fixes file-disappearance bug). Returns: ExecutionResult with captured outputs and any artifacts. @@ -170,11 +403,11 @@ async def run( loop = asyncio.get_running_loop() if DOCKER_SANDBOX_ENABLED: result = await loop.run_in_executor( - None, self._run_in_docker, code, session_id + None, self._run_in_docker, code, session_id, workspace_id ) else: result = await loop.run_in_executor( - None, self._run_sync, code, session_id + None, self._run_sync, code, session_id, workspace_id ) return result @@ -184,12 +417,15 @@ def _run_sync( self, code: str, session_id: str = _ANON_SESSION, + workspace_id: Optional[str] = None, ) -> ExecutionResult: """Synchronous subprocess execution — runs in a thread pool worker. Args: code: Python source code to execute. session_id: Session whose uploaded files to inject into the sandbox. + workspace_id: Optional workspace UUID used as a disk-file fallback + when the in-memory session cache is empty. Returns: ExecutionResult with captured outputs. @@ -203,7 +439,37 @@ def _run_sync( # Write only THIS session's files into the sandbox tmpdir. # get_session_files() returns a snapshot copy — safe to iterate. - for filename, content in get_session_files(session_id).items(): + session_files = get_session_files(session_id) + + # Workspace disk-file fallback: if the in-memory cache is empty + # (e.g. server restart, or workspace-upload path keyed differently) + # copy files from the workspace directory on disk. + if not session_files and workspace_id: + workspace_dir = os.path.join( + os.environ.get("WORKSPACE_FILES_DIR", "/workspace"), + workspace_id, + ) + if os.path.isdir(workspace_dir): + for fname in os.listdir(workspace_dir): + fpath = os.path.join(workspace_dir, fname) + if os.path.isfile(fpath): + try: + with open(fpath, "rb") as fh: + session_files[fname] = fh.read() + except Exception as exc: # pylint: disable=broad-except + logger.warning( + "[Executor] Could not read workspace file %s: %s", + fname, exc, + ) + if session_files: + logger.info( + "[Executor] Loaded %d file(s) from workspace disk fallback " + "(workspace_id=%s).", + len(session_files), + workspace_id, + ) + + for filename, content in session_files.items(): file_path = os.path.join(tmpdir, filename) try: with open(file_path, "wb") as fh: @@ -242,28 +508,29 @@ def _run_sync( ) try: - proc = subprocess.run( - [sys.executable, script_path], - capture_output=True, - text=True, - timeout=EXECUTION_TIMEOUT_SECONDS, + # BUG 1 fix: pass allowed_cwd=tmpdir so any absolute path + # outside the temp sandbox is caught before process spawn. + # _run_popen_capped now uses start_new_session + killpg for + # reliable process-tree teardown, and raises SandboxViolationError + # on timeout/constraint breach instead of raw exceptions. + stdout_text, stderr_text, returncode = _run_popen_capped( + args=[sys.executable, script_path], cwd=tmpdir, - env=_safe_env(), # SANITISED environment — no credentials + env=_safe_env(), + timeout=EXECUTION_TIMEOUT_SECONDS, + allowed_cwd=tmpdir, ) result = ExecutionResult( - stdout=proc.stdout, - stderr=proc.stderr, - returncode=proc.returncode, - ) - except subprocess.TimeoutExpired: - logger.warning( - "[Executor] Script timed out after %ds.", EXECUTION_TIMEOUT_SECONDS + stdout=stdout_text, + stderr=stderr_text, + returncode=returncode, ) + except SandboxViolationError as exc: + # Surface sandbox breaches clearly — do NOT retry these + logger.error("[Executor] Sandbox violation: %s", exc) result = ExecutionResult( stdout="", - stderr=( - f"Execution timed out after {EXECUTION_TIMEOUT_SECONDS} seconds." - ), + stderr=f"Sandbox violation: {exc}", returncode=1, ) except Exception as exc: # pylint: disable=broad-except @@ -292,6 +559,7 @@ def _run_in_docker( self, code: str, session_id: str = _ANON_SESSION, + workspace_id: Optional[str] = None, ) -> ExecutionResult: """Executes code inside a Docker container with network/resource limits. @@ -307,6 +575,8 @@ def _run_in_docker( Args: code: Python source code to execute. session_id: Session whose uploaded files to inject into the sandbox. + workspace_id: Optional workspace UUID; used as disk-file fallback + when the in-memory session cache is empty. Returns: ExecutionResult with captured outputs and any artifacts. @@ -334,7 +604,28 @@ def _run_in_docker( os.makedirs(outputs_dir, exist_ok=True) # Write session files to tmpdir so Docker can mount them - for filename, content in get_session_files(session_id).items(): + session_files = get_session_files(session_id) + + # Workspace disk-file fallback (mirrors _run_sync logic) + if not session_files and workspace_id: + workspace_dir = os.path.join( + os.environ.get("WORKSPACE_FILES_DIR", "/workspace"), + workspace_id, + ) + if os.path.isdir(workspace_dir): + for fname in os.listdir(workspace_dir): + fpath = os.path.join(workspace_dir, fname) + if os.path.isfile(fpath): + try: + with open(fpath, "rb") as fh: + session_files[fname] = fh.read() + except Exception as exc: # pylint: disable=broad-except + logger.warning( + "[Executor] Could not read workspace file %s: %s", + fname, exc, + ) + + for filename, content in session_files.items(): with open(os.path.join(tmpdir, filename), "wb") as fh: fh.write(content) diff --git a/backend/core/planner/planner_agent.py b/backend/core/planner/planner_agent.py index 3bc0ed3..89a8d71 100644 --- a/backend/core/planner/planner_agent.py +++ b/backend/core/planner/planner_agent.py @@ -37,6 +37,13 @@ class PlanStep(BaseModel): default="pending", description="Execution status: pending | done | failed.", ) + step_type: str = Field( + default="general", + description=( + "Step category — one of: load | filter | aggregate | visualise | " + "merge | clarify | general." + ), + ) class PlanOutput(BaseModel): @@ -66,11 +73,17 @@ class PlanOutput(BaseModel): " Visualization (plot/chart), or Insight (answer a question).\n" "- Tailor steps to the task type: e.g., include train/test split for ML, " " plt.savefig for Visualization, print() for Insight.\n" + "- Valid step_type values: load | filter | aggregate | visualise | " + " merge | clarify | general.\n" + "- Set step_type to 'merge' for any pd.merge() operation.\n" + "- Set step_type to 'clarify' ONLY when the join columns are ambiguous " + " (confidence < 0.5 for ALL candidates); do not proceed to coding in that case.\n" ) _INITIAL_PLAN_HUMAN = ( "USER QUERY:\n{query}\n\n" "DATA DESCRIPTION:\n{data_description}" + "{join_hints_block}" ) # --------------------------------------------------------------------------- @@ -151,6 +164,7 @@ async def create_plan( token_tracker: Optional[TokenTracker] = None, execution_output: str = "", completed_steps: Optional[List[Dict[str, Any]]] = None, + join_hints: str = "", ) -> List[Dict[str, Any]]: """Generates or extends the analysis plan. @@ -202,14 +216,17 @@ async def create_plan( f" Step {s['index'] + 1}: {s['description']} [{s.get('status', 'done')}]" for s in (completed_steps or []) ) or "(none yet)" - result: PlanOutput = await next_step_chain.ainvoke( - { - "query": query, - "data_description": data_description, - "completed_steps": completed_str, - "execution_output": execution_output[:2000], - }, - config=tracker_callback_config(token_tracker), + result: PlanOutput = await asyncio.wait_for( + next_step_chain.ainvoke( + { + "query": query, + "data_description": data_description, + "completed_steps": completed_str, + "execution_output": execution_output[:2000], + }, + config=tracker_callback_config(token_tracker), + ), + timeout=30.0, ) # Sequential mode should return 1 step; use the first if more steps = [result.steps[0].model_dump()] if result.steps else [] @@ -225,12 +242,34 @@ async def create_plan( if self._chain is None: # double-checked locking self._chain = built chain = self._chain - result = await chain.ainvoke( - { - "query": query, - "data_description": data_description, - }, - config=tracker_callback_config(token_tracker), + # Build optional join hints block for multi-file workspaces + if join_hints and join_hints.strip(): + join_hints_block = ( + "\n\nDETECTED JOIN KEYS:\n" + + join_hints + + "\n\nPLANNING RULES FOR MULTI-FILE WORKSPACES:\n" + "- When the user's query requires data from more than one file, " + "include a step of type 'merge' before any aggregation or filtering.\n" + "- A 'merge' step must specify in its description: " + "left_df, right_df, left_on, right_on, how (inner/left/right/outer).\n" + "- Prefer the highest-confidence join candidate unless the query " + "implies a specific column.\n" + "- If no confident candidate exists (all < 0.5), add a 'clarify' " + "step asking the user which columns to join on, and do NOT " + "proceed to coding." + ) + else: + join_hints_block = "" + result = await asyncio.wait_for( + chain.ainvoke( + { + "query": query, + "data_description": data_description, + "join_hints_block": join_hints_block, + }, + config=tracker_callback_config(token_tracker), + ), + timeout=30.0, ) steps = [s.model_dump() for s in result.steps] diff --git a/backend/db/add_eval_metrics_fk.sql b/backend/db/add_eval_metrics_fk.sql deleted file mode 100644 index e5cbf2d..0000000 --- a/backend/db/add_eval_metrics_fk.sql +++ /dev/null @@ -1,45 +0,0 @@ --- ============================================================ --- Migration: Add missing foreign-key constraints for eval tables --- Run this in the Supabase SQL editor ONCE. --- It is safe to re-run (idempotent via DO / IF NOT EXISTS guards). --- ============================================================ - --- ── eval_metrics.run_id → agent_runs.id ────────────────────────────────────── -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 FROM information_schema.table_constraints - WHERE constraint_type = 'FOREIGN KEY' - AND table_name = 'eval_metrics' - AND constraint_name = 'eval_metrics_run_id_fkey' - ) THEN - -- Clean up any orphaned records before applying the constraint - DELETE FROM eval_metrics WHERE run_id NOT IN (SELECT id FROM agent_runs); - - ALTER TABLE eval_metrics - ADD CONSTRAINT eval_metrics_run_id_fkey - FOREIGN KEY (run_id) REFERENCES agent_runs(id) ON DELETE CASCADE; - END IF; -END $$; - --- ── eval_steps.run_id → agent_runs.id ──────────────────────────────────────── -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 FROM information_schema.table_constraints - WHERE constraint_type = 'FOREIGN KEY' - AND table_name = 'eval_steps' - AND constraint_name = 'eval_steps_run_id_fkey' - ) THEN - -- Clean up any orphaned records before applying the constraint - DELETE FROM eval_steps WHERE run_id NOT IN (SELECT id FROM agent_runs); - - ALTER TABLE eval_steps - ADD CONSTRAINT eval_steps_run_id_fkey - FOREIGN KEY (run_id) REFERENCES agent_runs(id) ON DELETE CASCADE; - END IF; -END $$; - --- Reload PostgREST schema cache immediately so the new FK is visible --- to the auto-join syntax used by eval_store.list_run_metrics(). -NOTIFY pgrst, 'reload schema'; diff --git a/backend/db/create_eval_schema.sql b/backend/db/create_eval_schema.sql deleted file mode 100644 index a44278b..0000000 --- a/backend/db/create_eval_schema.sql +++ /dev/null @@ -1,72 +0,0 @@ --- ============================================================ --- DS-STAR Evaluation + Observability Schema --- Run the FULL contents of this file in the Supabase SQL editor. --- It is safe to re-run on an existing database (idempotent). --- ============================================================ - --- ── 1. eval_steps — one row per agent step per run ─────────────────────────── --- Captures every deterministic boundary inside the orchestrator loop. -create table if not exists eval_steps ( - id text primary key, -- uuid4 step_id - run_id text not null, -- FK → agent_runs.id - step_seq int not null, -- ordering within the run (0-based) - round_num int not null default 1, -- orchestrator round this step belongs to - agent_name text not null, -- Analyzer|Planner|Coder|Executor|Debugger|Verifier|Router|Finalizer - input_summary text, -- truncated input (≤ 512 chars) - output_summary text, -- truncated output (≤ 512 chars) - latency_ms int not null default 0, - retry_count int not null default 0, - error_type text, -- NULL = success - validation_passed boolean not null default true, - timestamp_iso text not null, - created_at timestamptz not null default now() -); - --- Indexes for common query patterns -create index if not exists eval_steps_run_id_idx on eval_steps (run_id); -create index if not exists eval_steps_agent_idx on eval_steps (agent_name); -create index if not exists eval_steps_error_idx on eval_steps (error_type) where error_type is not null; - --- ── 2. eval_metrics — one row per completed run (materialized) ─────────────── -create table if not exists eval_metrics ( - run_id text primary key, -- FK → agent_runs.id - -- Correctness - success_rate float not null default 0, -- 1.0 = verified, 0.0 = max rounds hit - exec_success_rate float not null default 0, -- fraction of rounds where execution succeeded - validation_pass_rate float not null default 0, -- step-level schema validation pass rate - -- Efficiency - total_retries int not null default 0, - avg_debug_depth float not null default 0, -- avg debug loop depth per failed exec - plan_step_count int not null default 0, -- steps in initial plan - final_step_count int not null default 0, -- steps in final plan (deviation indicator) - -- Per-agent latency breakdowns (ms) - analyzer_ms int not null default 0, - planner_ms int not null default 0, - coder_ms int not null default 0, - executor_ms int not null default 0, - debugger_ms int not null default 0, - verifier_ms int not null default 0, - router_ms int not null default 0, - finalizer_ms int not null default 0, - -- Classification - difficulty text not null default 'easy', -- easy | hard - mode text not null default 'live', -- live | batch - query_length int not null default 0, - created_at timestamptz not null default now() -); - -create index if not exists eval_metrics_difficulty_idx on eval_metrics (difficulty); -create index if not exists eval_metrics_mode_idx on eval_metrics (mode); -create index if not exists eval_metrics_created_idx on eval_metrics (created_at desc); - --- ── 3. RLS — allow all operations (same policy as agent_runs) ──────────────── -alter table eval_steps enable row level security; -alter table eval_metrics enable row level security; - -drop policy if exists "Allow all on eval_steps" on eval_steps; -create policy "Allow all on eval_steps" - on eval_steps for all using (true); - -drop policy if exists "Allow all on eval_metrics" on eval_metrics; -create policy "Allow all on eval_metrics" - on eval_metrics for all using (true); diff --git a/backend/db/migrations/003_workspace_files.sql b/backend/db/migrations/003_workspace_files.sql new file mode 100644 index 0000000..df512d0 --- /dev/null +++ b/backend/db/migrations/003_workspace_files.sql @@ -0,0 +1,122 @@ +-- Migration 003: workspace_files table +-- Enables multi-file workspace joins by storing per-file metadata alongside +-- the workspace session. Each row represents one uploaded file associated +-- with a workspace, including its parsed schema (columns + dtypes + samples) +-- so the schema_merger can infer join keys without re-reading the files. +-- +-- Backward-compat: A backfill INSERT creates one row (upload_order=1) for every +-- existing workspace that already stores a file_path column, preserving all +-- prior sessions without disruption. + +-- --------------------------------------------------------------------------- +-- Table +-- --------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS workspace_files ( + id uuid DEFAULT gen_random_uuid() PRIMARY KEY, + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + file_name text NOT NULL, + file_path text NOT NULL, + file_type text NOT NULL + CHECK (file_type IN ('csv', 'xlsx', 'parquet', 'md', 'json')), + row_count int, + schema_json jsonb NOT NULL DEFAULT '{}', + upload_order int NOT NULL DEFAULT 1, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- --------------------------------------------------------------------------- +-- Row-Level Security +-- --------------------------------------------------------------------------- + +ALTER TABLE workspace_files ENABLE ROW LEVEL SECURITY; + +-- Users can read their own files only. +CREATE POLICY workspace_files_select + ON workspace_files + FOR SELECT + USING (user_id = auth.uid()); + +-- Users can insert their own files only. +CREATE POLICY workspace_files_insert + ON workspace_files + FOR INSERT + WITH CHECK (user_id = auth.uid()); + +-- Users can delete their own files only. +CREATE POLICY workspace_files_delete + ON workspace_files + FOR DELETE + USING (user_id = auth.uid()); + +-- --------------------------------------------------------------------------- +-- Index +-- --------------------------------------------------------------------------- + +CREATE INDEX IF NOT EXISTS idx_workspace_files_lookup + ON workspace_files (workspace_id, user_id, upload_order ASC); + +-- --------------------------------------------------------------------------- +-- Backfill +-- Inserts one workspace_files row (upload_order=1) for each existing workspace +-- row that has a non-null file_path column. This keeps the schema_merger +-- aware of legacy single-file sessions without any manual intervention. +-- +-- If the workspaces table has no file_path column this statement is a no-op +-- because the SELECT returns no rows (column reference would fail at parse +-- time -- wrap in a DO block so it degrades gracefully). +-- --------------------------------------------------------------------------- + +DO $$ +BEGIN + -- Only run if the workspaces table has a file_path column (legacy schema). + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'workspaces' + AND column_name = 'file_path' + ) THEN + INSERT INTO workspace_files ( + workspace_id, + user_id, + file_name, + file_path, + file_type, + row_count, + schema_json, + upload_order, + created_at + ) + SELECT + w.id AS workspace_id, + w.user_id AS user_id, + COALESCE( + substring(w.file_path FROM '[^/]+$'), + 'unknown' + ) AS file_name, + w.file_path AS file_path, + -- Best-effort file type from extension; default to 'csv'. + LOWER(COALESCE( + substring(w.file_path FROM '\.([^.]+)$'), + 'csv' + )) AS file_type, + NULL AS row_count, + '{}'::jsonb AS schema_json, + 1 AS upload_order, + w.created_at AS created_at + FROM workspaces w + WHERE w.file_path IS NOT NULL + -- Skip workspaces that already have a workspace_files row. + AND NOT EXISTS ( + SELECT 1 + FROM workspace_files wf + WHERE wf.workspace_id = w.id + ); + + RAISE NOTICE 'workspace_files backfill complete.'; + ELSE + RAISE NOTICE 'workspaces.file_path column not present — backfill skipped (no-op).'; + END IF; +END; +$$; diff --git a/backend/db/migrations/004_workspace_files_file_size.sql b/backend/db/migrations/004_workspace_files_file_size.sql new file mode 100644 index 0000000..27140f7 --- /dev/null +++ b/backend/db/migrations/004_workspace_files_file_size.sql @@ -0,0 +1,22 @@ +-- Migration 004: workspace_files improvements +-- +-- 1. Add file_size column to track uploaded file size in bytes so the +-- frontend can display it without fetching file content. +-- 2. Widen the file_type CHECK constraint to include 'txt' and 'pdf' +-- which are accepted by the upload UI but were missing from the +-- original constraint, causing silent insert failures. + +-- Add file_size column (nullable for backward compat with existing rows) +ALTER TABLE workspace_files + ADD COLUMN IF NOT EXISTS file_size bigint; + +COMMENT ON COLUMN workspace_files.file_size + IS 'File size in bytes, populated at upload time.'; + +-- Drop the old restrictive CHECK and create a wider one +ALTER TABLE workspace_files + DROP CONSTRAINT IF EXISTS workspace_files_file_type_check; + +ALTER TABLE workspace_files + ADD CONSTRAINT workspace_files_file_type_check + CHECK (file_type IN ('csv', 'xlsx', 'xls', 'parquet', 'md', 'json', 'txt', 'pdf')); diff --git a/backend/eval/__init__.py b/backend/eval/__init__.py deleted file mode 100644 index e55f39b..0000000 --- a/backend/eval/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""DS-STAR Evaluation + Observability sidecar package. - -Non-invasive: the EvalLogger consumes existing SSE events that already -flow through agent_controller.py. Zero changes to any agent or orchestrator. -""" diff --git a/backend/eval/eval_engine.py b/backend/eval/eval_engine.py deleted file mode 100644 index 65c5591..0000000 --- a/backend/eval/eval_engine.py +++ /dev/null @@ -1,173 +0,0 @@ -"""EvalEngine — derives aggregated metrics from raw EvalStep traces. - -All computation is pure Python with no I/O so it is synchronous and -easily unit-testable. -""" - -from typing import Any, Dict, List - -from eval.schemas import EvalRunMetrics, EvalStep - - -# --------------------------------------------------------------------------- -# Per-run metric computation -# --------------------------------------------------------------------------- - -def compute_run_metrics( - steps: List[EvalStep], - run_meta: Dict[str, Any], - debug_depths: List[int], - initial_plan_count: int, - final_plan_count: int, - query_length: int = 0, - mode: str = "live", -) -> EvalRunMetrics: - """Derives an EvalRunMetrics from the captured steps and loop metadata. - - Args: - steps: All EvalStep objects captured for this run. - run_meta: The ``payload`` dict from the ``"metrics"`` SSE event. - debug_depths: List of debug loop depths per orchestrator round. - initial_plan_count: Steps in the initial plan (from ``plan_ready``). - final_plan_count: Steps in the final plan (from last ``plan_updated`` - or ``plan_ready`` if router never fired). - query_length: Character length of the original query. - mode: ``"live"`` or ``"batch"``. - - Returns: - Fully populated EvalRunMetrics instance. - """ - # Pull run_id from run_meta (the metrics SSE payload carries metrics sub-dict) - metrics_sub = run_meta.get("metrics", {}) - run_id: str = metrics_sub.get("run_id", run_meta.get("run_id", "")) - - # ── Correctness ─────────────────────────────────────────────────────── - final_status: str = metrics_sub.get("final_status", "max_rounds_reached") - success_rate = 1.0 if final_status == "completed" else 0.0 - - exec_steps = [s for s in steps if s.agent_name == "Executor"] - exec_success_rate = ( - sum(1 for s in exec_steps if s.validation_passed) / len(exec_steps) - if exec_steps else 0.0 - ) - - total_steps = len(steps) - validation_pass_rate = ( - sum(1 for s in steps if s.validation_passed) / total_steps - if total_steps else 1.0 - ) - - # ── Efficiency ──────────────────────────────────────────────────────── - total_retries = sum(s.retry_count for s in steps) - - avg_debug_depth = ( - sum(debug_depths) / len(debug_depths) if debug_depths else 0.0 - ) - - # ── Per-agent latency sums ───────────────────────────────────────────── - def _sum_ms(agent: str) -> int: - return sum(s.latency_ms for s in steps if s.agent_name == agent) - - # Pull per-round timings from the RoundMetric breakdown when available - per_round: List[Dict[str, Any]] = metrics_sub.get("per_round", []) - if per_round: - planner_ms = sum(r.get("planner_ms", 0) for r in per_round) - coder_ms = sum(r.get("coder_ms", 0) for r in per_round) - executor_ms = sum(r.get("executor_ms", 0) for r in per_round) - verifier_ms = sum(r.get("verifier_ms", 0) for r in per_round) - router_ms = sum(r.get("router_ms", 0) for r in per_round) - else: - # Fallback: derive from step-level captures - planner_ms = _sum_ms("Planner") - coder_ms = _sum_ms("Coder") - executor_ms = _sum_ms("Executor") - verifier_ms = _sum_ms("Verifier") - router_ms = _sum_ms("Router") - - analyzer_ms = _sum_ms("Analyzer") - debugger_ms = _sum_ms("Debugger") - finalizer_ms = _sum_ms("Finalizer") - - # ── Difficulty ───────────────────────────────────────────────────────── - difficulty = run_meta.get("complexity", metrics_sub.get("complexity", "easy")) - - return EvalRunMetrics( - run_id=run_id, - success_rate=round(success_rate, 4), - exec_success_rate=round(exec_success_rate, 4), - validation_pass_rate=round(validation_pass_rate, 4), - total_retries=total_retries, - avg_debug_depth=round(avg_debug_depth, 2), - plan_step_count=initial_plan_count, - final_step_count=final_plan_count, - analyzer_ms=analyzer_ms, - planner_ms=planner_ms, - coder_ms=coder_ms, - executor_ms=executor_ms, - debugger_ms=debugger_ms, - verifier_ms=verifier_ms, - router_ms=router_ms, - finalizer_ms=finalizer_ms, - difficulty=difficulty, - mode=mode, - query_length=query_length, - ) - - -# --------------------------------------------------------------------------- -# Cross-run agent summary (for dashboard /agents endpoint) -# --------------------------------------------------------------------------- - -def compute_agent_summary(rows: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: - """Aggregates per-agent statistics across multiple eval_metrics rows. - - Args: - rows: Rows from the ``eval_metrics`` table (list of dicts). - - Returns: - Dict keyed by agent_name with keys: - ``avg_latency_ms``, ``total_calls``, ``failure_rate``, - ``avg_retry_rate``. - """ - agents = [ - "Analyzer", "Planner", "Coder", "Executor", - "Debugger", "Verifier", "Router", "Finalizer", - ] - latency_col = { - "Analyzer": "analyzer_ms", - "Planner": "planner_ms", - "Coder": "coder_ms", - "Executor": "executor_ms", - "Debugger": "debugger_ms", - "Verifier": "verifier_ms", - "Router": "router_ms", - "Finalizer": "finalizer_ms", - } - summary: Dict[str, Dict[str, Any]] = {} - n = len(rows) or 1 - - for agent in agents: - col = latency_col[agent] - latencies = [r.get(col, 0) for r in rows if r.get(col, 0) > 0] - avg_lat = round(sum(latencies) / len(latencies), 1) if latencies else 0 - - # Executor failure rate = 1 - exec_success_rate - if agent == "Executor": - if len(rows) == 0: - failure_rate = 0.0 - else: - failure_rate = round( - 1.0 - (sum(r.get("exec_success_rate", 0) for r in rows) / n), 4 - ) - elif agent in ("Verifier", "Analyzer", "Planner", "Coder", "Finalizer"): - # Proxy: fraction of runs the agent had retries - failure_rate = 0.0 # refined below via step data if available - else: - failure_rate = 0.0 - - summary[agent] = { - "avg_latency_ms": avg_lat, - "failure_rate": failure_rate, - "total_calls": len(latencies), - } - return summary diff --git a/backend/eval/eval_logger.py b/backend/eval/eval_logger.py deleted file mode 100644 index 2a0c081..0000000 --- a/backend/eval/eval_logger.py +++ /dev/null @@ -1,333 +0,0 @@ -"""EvalLogger — the passive SSE-event sidecar. - -Consumes the exact same event dicts that agent_controller.py already -yields as SSE. Zero changes to agents or orchestrator required. - -Usage (inside agent_controller.handle_agent_run):: - - eval_logger = EvalLogger(run_id=run_id, query=query) - - async for event in orchestrator.run(...): - eval_logger.ingest(event) # ← passive observation - yield f"data: {json.dumps(event)}\\n\\n" - - await eval_logger.finalize() # ← flush to Supabase - -Mapping of SSE event-types to EvalStep captures: - - analysis_complete → Analyzer step - plan_ready → Planner step (captures initial plan length) - code_ready → Coder step - execution_result → Executor step (error_type set on failure) - debug_applied → Debugger step (increments retry_count) - verification_result→ Verifier step - plan_updated → Router step - finalized → Finalizer step - metrics → triggers finalize() automatically - -All other event types (round_start, retrying, warning, etc.) are observed -but do not produce new steps — they update state on the *current* open step. -""" - -import datetime -import logging -import uuid -from typing import Any, Dict, List, Optional - -from eval.schemas import EvalRunMetrics, EvalStep -from eval.eval_engine import compute_run_metrics - -logger = logging.getLogger("uvicorn.info") - -# --------------------------------------------------------------------------- -# Agents that produce an EvalStep row -# --------------------------------------------------------------------------- - -_STEP_EVENTS = { - "analysis_complete": "Analyzer", - "plan_ready": "Planner", - "code_ready": "Coder", - "execution_result": "Executor", - "debug_applied": "Debugger", - "verification_result": "Verifier", - "plan_updated": "Router", - "finalized": "Finalizer", -} - - -def _trunc(text: Any, limit: int = 512) -> str: - """Truncates any value to a safely storable string.""" - s = str(text) if text is not None else "" - return s[:limit] - - -def _now_iso() -> str: - return datetime.datetime.utcnow().isoformat() + "Z" - - -# --------------------------------------------------------------------------- -# EvalLogger -# --------------------------------------------------------------------------- - -class EvalLogger: - """Stateful accumulator for one DS-STAR agent run. - - Thread-safety: single-threaded SSE generator — no locks needed. - - Attributes: - run_id: Matches the agent_runs PK. - query: Original user query (for query_length metric). - """ - - def __init__(self, run_id: str, query: str = "") -> None: - self.run_id = run_id - self.query = query - self._steps: List[EvalStep] = [] - self._step_seq = 0 - self._current_round = 1 - - # Retry tracking — counts retrying events before a step closes - self._current_retry_count = 0 - - # Debug depth tracking per round - self._debug_depths: List[int] = [] - self._current_debug_depth = 0 - - # Final plan size (captured from plan_updated or plan_ready) - self._initial_plan_count = 0 - self._final_plan_count = 0 - - # Run-level metadata captured from the "metrics" event - self._run_meta: Dict[str, Any] = {} - - # Mode — overridden to "batch" only by external callers - self.mode: str = "live" - - logger.debug("[EvalLogger] Initialized for run_id=%s", run_id) - - # ----------------------------------------------------------------------- - # Public: ingest one event - # ----------------------------------------------------------------------- - - def ingest(self, event: Dict[str, Any]) -> None: - """Observes one SSE event emitted by the orchestrator. - - Builds EvalStep rows at deterministic boundaries and tracks - state needed for loop-level metrics. - - Args: - event: The raw event dict ``{event: str, payload: dict}``. - """ - event_type: str = event.get("event", "") - payload: Dict[str, Any] = event.get("payload", {}) - - # ── Track current round ──────────────────────────────────────────── - if event_type == "round_start": - self._current_round = payload.get("round", self._current_round) - self._current_debug_depth = 0 - self._current_retry_count = 0 - return - - # ── Accumulate retries for the next step ─────────────────────────── - if event_type == "retrying": - self._current_retry_count += 1 - return - - # ── Debug depth tracking ─────────────────────────────────────────── - if event_type == "debug_applied": - self._current_debug_depth += 1 - - # ── Finalise debug depth when we leave the debug sub-loop ───────── - if event_type == "verification_result": - self._debug_depths.append(self._current_debug_depth) - self._current_debug_depth = 0 - - # ── Capture plan sizes ───────────────────────────────────────────── - if event_type == "plan_ready": - self._initial_plan_count = len(payload.get("steps", [])) - self._final_plan_count = self._initial_plan_count - - if event_type == "plan_updated": - self._final_plan_count = len(payload.get("steps", [])) - - # ── Capture run-level metadata from metrics event ────────────────── - if event_type == "metrics": - self._run_meta = payload - return # no step row for the metrics event itself - - # ── Build EvalStep at each deterministic boundary ───────────────── - agent_name = _STEP_EVENTS.get(event_type) - if agent_name is None: - return # not a step-producing event - - step = self._build_step(agent_name, event_type, payload) - self._steps.append(step) - self._step_seq += 1 - - # Reset retry counter after consuming it - self._current_retry_count = 0 - - # ----------------------------------------------------------------------- - # Public: flush to Supabase - # ----------------------------------------------------------------------- - - async def finalize(self) -> None: - """Flushes accumulated steps and computed run metrics to Supabase. - - Called once, after the SSE stream ends (in the finally block of - handle_agent_run). Failures are logged as warnings — never raised. - """ - if not self.run_id: - return - - try: - from eval.eval_store import upsert_steps, upsert_run_metrics # pylint: disable=import-outside-toplevel - - # ── Persist step traces ──────────────────────────────────────── - await upsert_steps(self._steps) - - # ── Compute and persist aggregated run metrics ───────────────── - run_metrics = compute_run_metrics( - steps=self._steps, - run_meta=self._run_meta, - debug_depths=self._debug_depths, - initial_plan_count=self._initial_plan_count, - final_plan_count=self._final_plan_count, - query_length=len(self.query), - mode=self.mode, - ) - await upsert_run_metrics(run_metrics) - - logger.info( - "[EvalLogger] Flushed %d steps + metrics for run_id=%s", - len(self._steps), - self.run_id, - ) - except Exception as exc: # pylint: disable=broad-except - logger.warning( - "[EvalLogger] Could not flush eval data for run_id=%s: %s", - self.run_id, - exc, - ) - - # ----------------------------------------------------------------------- - # Private helpers - # ----------------------------------------------------------------------- - - def _build_step( - self, - agent_name: str, - event_type: str, - payload: Dict[str, Any], - ) -> EvalStep: - """Constructs an EvalStep from a payload dict. - - Each event type exposes slightly different payload keys — this - method normalises them into the common EvalStep schema. - """ - latency_ms = self._extract_latency(event_type, payload) - error_type = self._extract_error(event_type, payload) - input_summary = self._extract_input(event_type, payload) - output_summary = self._extract_output(event_type, payload) - validation_passed = self._extract_validation(event_type, payload) - - return EvalStep( - step_id=uuid.uuid4().hex, - run_id=self.run_id, - step_seq=self._step_seq, - round_num=self._current_round, - agent_name=agent_name, - input_summary=input_summary, - output_summary=output_summary, - latency_ms=latency_ms, - retry_count=self._current_retry_count, - error_type=error_type, - validation_passed=validation_passed, - timestamp_iso=_now_iso(), - ) - - @staticmethod - def _extract_latency(event_type: str, payload: Dict[str, Any]) -> int: - """Pulls the most specific latency key from the payload.""" - keys = { - "code_ready": "coder_ms", - "execution_result": "executor_ms", - "verification_result": "verifier_ms", - "plan_updated": "router_ms", - "debug_applied": "debug_ms", - } - key = keys.get(event_type) - if key: - return int(payload.get(key, 0)) - return 0 - - @staticmethod - def _extract_error(event_type: str, payload: Dict[str, Any]) -> Optional[str]: - """Returns an error classification string or None on success.""" - if event_type == "execution_result": - if not payload.get("success", True): - stderr: str = payload.get("stderr", "") - # Classify by first token of error line - for line in stderr.splitlines(): - line = line.strip() - if line and ("Error" in line or "Exception" in line): - return line.split(":")[0].strip()[:80] - return "ExecutionError" - if event_type == "debug_applied": - return payload.get("error_type") or None - return None - - @staticmethod - def _extract_input(event_type: str, payload: Dict[str, Any]) -> str: - if event_type == "plan_ready": - steps = payload.get("steps", []) - return _trunc(f"{len(steps)} plan steps") - if event_type == "code_ready": - code = payload.get("code", "") - return _trunc(f"round={payload.get('round',1)} prev_code={len(code)} chars") - if event_type == "execution_result": - return _trunc(f"round={payload.get('round',1)}") - if event_type == "debug_applied": - return _trunc(payload.get("error_type", "")) - if event_type == "verification_result": - return _trunc(payload.get("reason", "")) - if event_type == "plan_updated": - return _trunc(f"action={payload.get('action','')}") - return "" - - @staticmethod - def _extract_output(event_type: str, payload: Dict[str, Any]) -> str: - if event_type == "analysis_complete": - desc = payload.get("data_description", "") - return _trunc(desc) - if event_type == "plan_ready": - steps = payload.get("steps", []) - return _trunc(", ".join( - s.get("description", "")[:60] for s in steps[:5] - )) - if event_type == "code_ready": - return _trunc(payload.get("code", "")[:512]) - if event_type == "execution_result": - stdout = payload.get("stdout", "") - stderr = payload.get("stderr", "") - return _trunc(stdout or stderr) - if event_type == "debug_applied": - return _trunc(payload.get("fix_summary", "")) - if event_type == "verification_result": - conf = payload.get("confidence", 0) - suf = payload.get("is_sufficient", False) - return _trunc(f"sufficient={suf} confidence={conf:.2f}") - if event_type == "plan_updated": - steps = payload.get("steps", []) - return _trunc(f"{len(steps)} updated steps") - if event_type == "finalized": - return _trunc(payload.get("headline", "") or payload.get("formatted_output", "")) - return "" - - @staticmethod - def _extract_validation(event_type: str, payload: Dict[str, Any]) -> bool: - if event_type == "execution_result": - return bool(payload.get("success", True)) - if event_type == "verification_result": - return bool(payload.get("is_sufficient", False)) - return True diff --git a/backend/eval/eval_routes.py b/backend/eval/eval_routes.py deleted file mode 100644 index 56b0554..0000000 --- a/backend/eval/eval_routes.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Eval Dashboard API — read-only endpoints served under /api/eval/*. - -All routes are GET except one POST for triggering offline eval (deferred). -No authentication required — same open policy as the rest of the API. - -Endpoints: - GET /api/eval/overview — system-level KPIs - GET /api/eval/agents — per-agent latency + failure rates - GET /api/eval/debug-loop — debug loop depth, error distribution - GET /api/eval/runs — paginated run list with metrics - GET /api/eval/runs/{run_id}/trace — step-by-step trace for one run -""" - -import logging -from typing import Any, Dict, List, Optional - -from fastapi import APIRouter, Depends, HTTPException, Query -from middleware.auth import AuthUser, get_current_user - -logger = logging.getLogger("uvicorn.info") - -eval_router = APIRouter(tags=["eval"]) - - -# --------------------------------------------------------------------------- -# System Overview -# --------------------------------------------------------------------------- - -@eval_router.get("/overview") -async def eval_overview( - auth: AuthUser = Depends(get_current_user), -) -> Dict[str, Any]: - """Returns aggregated system-level KPIs from all recorded runs. - - Returns: - Dict with ``total_runs``, ``success_rate`` (%), - ``avg_latency_ms``, ``avg_retries``, ``easy_count``, ``hard_count``. - """ - try: - from eval.eval_store import get_overview_stats # pylint: disable=import-outside-toplevel - return await get_overview_stats() - except Exception as exc: # pylint: disable=broad-except - logger.error("[EvalRoutes] overview error: %s", exc) - raise HTTPException(status_code=500, detail=str(exc)) from exc - - -# --------------------------------------------------------------------------- -# Agent Performance -# --------------------------------------------------------------------------- - -@eval_router.get("/agents") -async def eval_agents( - auth: AuthUser = Depends(get_current_user), -) -> List[Dict[str, Any]]: - """Returns per-agent performance metrics aggregated across all runs. - - Returns: - List of ``{agent_name, avg_latency_ms, failure_rate, total_calls}``. - """ - try: - from eval.eval_store import get_agent_stats # pylint: disable=import-outside-toplevel - return await get_agent_stats() - except Exception as exc: # pylint: disable=broad-except - logger.error("[EvalRoutes] agents error: %s", exc) - raise HTTPException(status_code=500, detail=str(exc)) from exc - - -# --------------------------------------------------------------------------- -# Debug Loop Analysis -# --------------------------------------------------------------------------- - -@eval_router.get("/debug-loop") -async def eval_debug_loop( - auth: AuthUser = Depends(get_current_user), -) -> Dict[str, Any]: - """Returns debug loop depth and error type distribution. - - Returns: - Dict with ``avg_debug_depth``, ``avg_retries``, - ``retry_success_ratio``, ``error_type_distribution``. - """ - try: - from eval.eval_store import get_debug_loop_stats # pylint: disable=import-outside-toplevel - return await get_debug_loop_stats() - except Exception as exc: # pylint: disable=broad-except - logger.error("[EvalRoutes] debug-loop error: %s", exc) - raise HTTPException(status_code=500, detail=str(exc)) from exc - - -# --------------------------------------------------------------------------- -# Run List -# --------------------------------------------------------------------------- - -@eval_router.get("/runs") -async def eval_runs( - limit: int = Query(default=50, ge=1, le=100), - difficulty: Optional[str] = Query(default=None, pattern="^(easy|hard)$"), - mode: Optional[str] = Query(default=None, pattern="^(live|batch)$"), - auth: AuthUser = Depends(get_current_user), -) -> List[Dict[str, Any]]: - """Returns a paginated list of runs with their computed eval metrics. - - Query params: - limit: Max rows (1–100, default 50). - difficulty: Filter by task difficulty (``easy`` | ``hard``). - mode: Filter by eval mode (``live`` | ``batch``). - - Returns: - List of eval_metrics rows joined with agent_runs metadata. - """ - try: - from eval.eval_store import list_run_metrics # pylint: disable=import-outside-toplevel - return await list_run_metrics(limit=limit, difficulty=difficulty, mode=mode) - except Exception as exc: # pylint: disable=broad-except - logger.error("[EvalRoutes] runs error: %s", exc) - raise HTTPException(status_code=500, detail=str(exc)) from exc - - -# --------------------------------------------------------------------------- -# Run Trace -# --------------------------------------------------------------------------- - -@eval_router.get("/runs/{run_id}/trace") -async def eval_run_trace( - run_id: str, - auth: AuthUser = Depends(get_current_user), -) -> Dict[str, Any]: - """Returns the step-by-step eval trace and metadata for one run. - - Args: - run_id: The agent_runs.id UUID. - - Returns: - Dict with ``run_id``, ``steps`` (list), and ``agent_runs`` metadata. - """ - try: - from eval.eval_store import list_steps_for_run # pylint: disable=import-outside-toplevel - from services.supabase_service import get_agent_run # pylint: disable=import-outside-toplevel - - steps, run = await _gather( - list_steps_for_run(run_id), - get_agent_run(run_id), - ) - - if not run: - raise HTTPException( - status_code=404, - detail=f"Run '{run_id}' not found in agent_runs.", - ) - - return {"run_id": run_id, "run": run, "steps": steps} - except HTTPException: - raise - except Exception as exc: # pylint: disable=broad-except - logger.error("[EvalRoutes] trace error: %s", exc) - raise HTTPException(status_code=500, detail=str(exc)) from exc - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -import asyncio # noqa: E402 (import after main defs is intentional) - - -async def _gather(*coros): - return await asyncio.gather(*coros) diff --git a/backend/eval/eval_store.py b/backend/eval/eval_store.py deleted file mode 100644 index 65d93ae..0000000 --- a/backend/eval/eval_store.py +++ /dev/null @@ -1,376 +0,0 @@ -"""EvalStore — Supabase persistence for the evaluation sidecar. - -Mirrors the async executor pattern from services/supabase_service.py: -all blocking supabase-py calls are dispatched via run_in_executor so the -FastAPI event loop is never blocked during SSE streaming. -""" - -import asyncio -import logging -from typing import Any, Dict, List - -from eval.schemas import EvalRunMetrics, EvalStep - -logger = logging.getLogger("uvicorn.info") - - -# --------------------------------------------------------------------------- -# Internal helper — reuse the same Supabase client from services -# --------------------------------------------------------------------------- - -def _get_client(service_role: bool = False): - from services.supabase_service import get_supabase_client, get_service_role_client # pylint: disable=import-outside-toplevel - if service_role: - return get_service_role_client() - return get_supabase_client() - - -def _is_auth_failed() -> bool: - """Returns the circuit-breaker flag from supabase_service.""" - try: - from services import supabase_service # pylint: disable=import-outside-toplevel - return supabase_service._supabase_auth_failed # pylint: disable=protected-access - except Exception: # pylint: disable=broad-except - return False - - -# --------------------------------------------------------------------------- -# Write operations -# --------------------------------------------------------------------------- - -async def upsert_steps(steps: List[EvalStep]) -> None: - """Batch-inserts EvalStep rows into the ``eval_steps`` table. - - Uses upsert (on_conflict=id) so re-runs of finalize() are safe. - - Args: - steps: Collected EvalStep objects for the run. - """ - if not steps: - return - - records = [ - { - "id": s.step_id, - "run_id": s.run_id, - "step_seq": s.step_seq, - "round_num": s.round_num, - "agent_name": s.agent_name, - "input_summary": s.input_summary[:512], - "output_summary": s.output_summary[:512], - "latency_ms": s.latency_ms, - "retry_count": s.retry_count, - "error_type": s.error_type, - "validation_passed": s.validation_passed, - "timestamp_iso": s.timestamp_iso, - } - for s in steps - ] - - def _sync() -> None: - if _is_auth_failed(): - return # Circuit open — skip silently - client = _get_client(service_role=True) - try: - client.table("eval_steps").upsert(records).execute() - logger.info("[EvalStore] Upserted %d eval_steps rows", len(records)) - except Exception as exc: # pylint: disable=broad-except - logger.warning("[EvalStore] Could not upsert eval_steps: %s", exc) - - await asyncio.get_running_loop().run_in_executor(None, _sync) - - -async def upsert_run_metrics(metrics: EvalRunMetrics) -> None: - """Upserts a single EvalRunMetrics row into ``eval_metrics``. - - Args: - metrics: Computed metrics for the run. - """ - record = { - "run_id": metrics.run_id, - "success_rate": metrics.success_rate, - "exec_success_rate": metrics.exec_success_rate, - "validation_pass_rate": metrics.validation_pass_rate, - "total_retries": metrics.total_retries, - "avg_debug_depth": metrics.avg_debug_depth, - "plan_step_count": metrics.plan_step_count, - "final_step_count": metrics.final_step_count, - "analyzer_ms": metrics.analyzer_ms, - "planner_ms": metrics.planner_ms, - "coder_ms": metrics.coder_ms, - "executor_ms": metrics.executor_ms, - "debugger_ms": metrics.debugger_ms, - "verifier_ms": metrics.verifier_ms, - "router_ms": metrics.router_ms, - "finalizer_ms": metrics.finalizer_ms, - "difficulty": metrics.difficulty, - "mode": metrics.mode, - "query_length": metrics.query_length, - } - - def _sync() -> None: - if _is_auth_failed(): - return # Circuit open — skip silently - client = _get_client(service_role=True) - try: - client.table("eval_metrics").upsert(record).execute() - logger.info( - "[EvalStore] Upserted eval_metrics for run_id=%s success_rate=%.2f", - metrics.run_id, - metrics.success_rate, - ) - except Exception as exc: # pylint: disable=broad-except - logger.warning("[EvalStore] Could not upsert eval_metrics: %s", exc) - - await asyncio.get_running_loop().run_in_executor(None, _sync) - - -# --------------------------------------------------------------------------- -# Read operations (used by eval_routes.py) -# --------------------------------------------------------------------------- - -async def list_steps_for_run(run_id: str) -> List[Dict[str, Any]]: - """Fetches all step rows for a single run, ordered by step_seq.""" - - def _sync() -> List[Dict[str, Any]]: - client = _get_client() - try: - resp = ( - client.table("eval_steps") - .select("*") - .eq("run_id", run_id) - .order("step_seq") - .execute() - ) - return resp.data or [] - except Exception as exc: # pylint: disable=broad-except - logger.warning("[EvalStore] Could not list eval_steps: %s", exc) - return [] - - return await asyncio.get_running_loop().run_in_executor(None, _sync) - - -async def list_run_metrics( - limit: int = 50, - difficulty: str | None = None, - mode: str | None = None, -) -> List[Dict[str, Any]]: - """Fetches eval_metrics rows enriched with basic agent_run info. - - Uses a two-query approach instead of the PostgREST implicit-join syntax - ``agent_runs(...)`` which requires a declared FK constraint in the schema - cache (PGRST200 if the constraint is missing). This makes the function - resilient both before and after the FK migration is applied. - - Args: - limit: Max rows (1–100). - difficulty: Optional filter — ``"easy"`` or ``"hard"``. - mode: Optional filter — ``"live"`` or ``"batch"``. - - Returns: - List of eval_metrics rows ordered newest first, each optionally - enriched with ``agent_runs`` sub-keys. - """ - - def _sync() -> List[Dict[str, Any]]: - client = _get_client() - - # ── 1. Fetch eval_metrics (no join) ─────────────────────────────────── - try: - q = ( - client.table("eval_metrics") - .select("*") - .order("created_at", desc=True) - .limit(limit) - ) - if difficulty: - q = q.eq("difficulty", difficulty) - if mode: - q = q.eq("mode", mode) - resp = q.execute() - rows: List[Dict[str, Any]] = resp.data or [] - except Exception as exc: # pylint: disable=broad-except - logger.warning("[EvalStore] Could not list eval_metrics: %s", exc) - return [] - - if not rows: - return rows - - # ── 2. Enrich with agent_runs data in a separate query ──────────────── - run_ids = [r["run_id"] for r in rows if r.get("run_id")] - run_map: Dict[str, Any] = {} - if run_ids: - try: - runs_resp = ( - client.table("agent_runs") - .select("id, query, status, created_at, completed_at, rounds") - .in_("id", run_ids) - .execute() - ) - for run in (runs_resp.data or []): - run_map[run["id"]] = run - except Exception as exc: # pylint: disable=broad-except - logger.warning( - "[EvalStore] Could not enrich eval_metrics with agent_runs: %s", exc - ) - - # Merge: embed run info under the "agent_runs" key to preserve the - # same response shape that the frontend / eval_routes already expect. - for row in rows: - row["agent_runs"] = run_map.get(row.get("run_id", ""), {}) - - return rows - - return await asyncio.get_running_loop().run_in_executor(None, _sync) - - -async def get_overview_stats() -> Dict[str, Any]: - """Computes system-level overview KPIs from eval_metrics. - - Returns: - Dict with ``total_runs``, ``success_rate``, ``avg_latency_ms``, - ``avg_retries``, ``easy_count``, ``hard_count``. - """ - - def _sync() -> Dict[str, Any]: - client = _get_client() - try: - resp = ( - client.table("eval_metrics") - .select( - "success_rate, total_retries, difficulty, " - "analyzer_ms, planner_ms, coder_ms, executor_ms, " - "verifier_ms, router_ms, debugger_ms, finalizer_ms" - ) - .execute() - ) - rows = resp.data or [] - if not rows: - return _empty_overview() - - n = len(rows) - avg_success = sum(r["success_rate"] for r in rows) / n - avg_retries = sum(r["total_retries"] for r in rows) / n - avg_latency = sum( - (r.get("coder_ms", 0) + r.get("executor_ms", 0) + - r.get("verifier_ms", 0) + r.get("planner_ms", 0)) - for r in rows - ) / n - easy = sum(1 for r in rows if r.get("difficulty") == "easy") - hard = n - easy - - return { - "total_runs": n, - "success_rate": round(avg_success * 100, 1), - "avg_latency_ms": round(avg_latency), - "avg_retries": round(avg_retries, 2), - "easy_count": easy, - "hard_count": hard, - } - except Exception as exc: # pylint: disable=broad-except - logger.warning("[EvalStore] overview_stats error: %s", exc) - return _empty_overview() - - return await asyncio.get_running_loop().run_in_executor(None, _sync) - - -def _empty_overview() -> Dict[str, Any]: - return { - "total_runs": 0, - "success_rate": 0.0, - "avg_latency_ms": 0, - "avg_retries": 0.0, - "easy_count": 0, - "hard_count": 0, - } - - -async def get_agent_stats() -> List[Dict[str, Any]]: - """Returns per-agent latency averages derived from eval_metrics rows. - - Returns: - List of dicts ``{agent_name, avg_latency_ms, total_calls}``. - """ - - def _sync() -> List[Dict[str, Any]]: - client = _get_client() - try: - resp = ( - client.table("eval_metrics") - .select( - "analyzer_ms, planner_ms, coder_ms, executor_ms, " - "debugger_ms, verifier_ms, router_ms, finalizer_ms, " - "exec_success_rate, avg_debug_depth" - ) - .execute() - ) - rows = resp.data or [] - except Exception as exc: # pylint: disable=broad-except - logger.warning("[EvalStore] agent_stats error: %s", exc) - rows = [] - - from eval.eval_engine import compute_agent_summary # pylint: disable=import-outside-toplevel - summary = compute_agent_summary(rows) - return [ - {"agent_name": name, **stats} - for name, stats in summary.items() - ] - - return await asyncio.get_running_loop().run_in_executor(None, _sync) - - -async def get_debug_loop_stats() -> Dict[str, Any]: - """Returns debug loop aggregate statistics. - - Returns: - Dict with ``avg_debug_depth``, ``error_type_distribution``, - and ``retry_success_ratio``. - """ - - def _sync() -> Dict[str, Any]: - client = _get_client() - try: - # Aggregate debug depth from eval_metrics - resp_m = ( - client.table("eval_metrics") - .select("avg_debug_depth, total_retries, exec_success_rate") - .execute() - ) - rows_m = resp_m.data or [] - - # Error type distribution from eval_steps - resp_s = ( - client.table("eval_steps") - .select("error_type") - .not_.is_("error_type", "null") - .execute() - ) - rows_s = resp_s.data or [] - except Exception as exc: # pylint: disable=broad-except - logger.warning("[EvalStore] debug_loop_stats error: %s", exc) - rows_m, rows_s = [], [] - - n = len(rows_m) or 1 - avg_depth = sum(r.get("avg_debug_depth", 0) for r in rows_m) / n - avg_retries = sum(r.get("total_retries", 0) for r in rows_m) / n - avg_exec_success = sum(r.get("exec_success_rate", 0) for r in rows_m) / n - - # Build error type distribution - error_dist: Dict[str, int] = {} - for r in rows_s: - et = r.get("error_type") or "Unknown" - error_dist[et] = error_dist.get(et, 0) + 1 - - return { - "avg_debug_depth": round(avg_depth, 2), - "avg_retries": round(avg_retries, 2), - "retry_success_ratio": round(avg_exec_success, 4), - "error_type_distribution": [ - {"error_type": et, "count": cnt} - for et, cnt in sorted( - error_dist.items(), key=lambda x: -x[1] - )[:10] - ], - } - - return await asyncio.get_running_loop().run_in_executor(None, _sync) diff --git a/backend/eval/schemas.py b/backend/eval/schemas.py deleted file mode 100644 index fbcfae4..0000000 --- a/backend/eval/schemas.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Pydantic schemas for the DS-STAR evaluation sidecar. - -These models mirror the ``eval_steps`` and ``eval_metrics`` Supabase tables -defined in migrations/create_eval_schema.sql. -""" - -from typing import Optional - -from pydantic import BaseModel, Field - - -# --------------------------------------------------------------------------- -# Step-level capture -# --------------------------------------------------------------------------- - -class EvalStep(BaseModel): - """One captured boundary event inside the DS-STAR orchestrator loop. - - Attributes: - step_id: UUID4 primary key for the eval_steps row. - run_id: Matches agent_runs.id. - step_seq: 0-based ordering within the run. - round_num: 1-based orchestrator round this step belongs to. - agent_name: One of Analyzer | Planner | Coder | Executor | - Debugger | Verifier | Router | Finalizer. - input_summary: Truncated input context (≤ 512 chars). - output_summary: Truncated output / result (≤ 512 chars). - latency_ms: Wall-clock ms spent in this step. - retry_count: Number of tenacity retries consumed. - error_type: Populated only on failure (e.g. ``"SyntaxError"``). - validation_passed: Whether the step output passed schema validation. - timestamp_iso: ISO-8601 wall-clock timestamp at step start. - """ - - step_id: str - run_id: str - step_seq: int = Field(ge=0) - round_num: int = Field(default=1, ge=1) - agent_name: str - input_summary: str = "" - output_summary: str = "" - latency_ms: int = Field(default=0, ge=0) - retry_count: int = Field(default=0, ge=0) - error_type: Optional[str] = None - validation_passed: bool = True - timestamp_iso: str - - -# --------------------------------------------------------------------------- -# Run-level computed metrics -# --------------------------------------------------------------------------- - -class EvalRunMetrics(BaseModel): - """Aggregated evaluation metrics for a single DS-STAR run. - - Computed by EvalEngine from the collected EvalStep list after the run - completes. Persisted to the ``eval_metrics`` Supabase table. - - Attributes: - run_id: Matches agent_runs.id. - success_rate: 1.0 if Verifier approved; 0.0 if max rounds hit. - exec_success_rate: Fraction of rounds where execution succeeded. - validation_pass_rate: Fraction of steps that passed schema checks. - total_retries: Sum of retry_count across all steps. - avg_debug_depth: Average Debugger invocations per failed exec round. - plan_step_count: Steps in the initial plan. - final_step_count: Steps in the final plan. - analyzer_ms: Latency for the Analyzer stage. - planner_ms: Latency for the Planner stage (round 1). - coder_ms: Cumulative Coder latency across all rounds. - executor_ms: Cumulative Executor latency across all rounds. - debugger_ms: Cumulative Debugger latency across all rounds. - verifier_ms: Cumulative Verifier latency across all rounds. - router_ms: Cumulative Router latency across all rounds. - finalizer_ms: Latency for the Finalizer stage. - difficulty: Task difficulty tag — ``"easy"`` or ``"hard"``. - mode: ``"live"`` (production) or ``"batch"`` (offline eval). - query_length: Character length of the original query. - """ - - run_id: str - # Correctness - success_rate: float = Field(default=0.0, ge=0.0, le=1.0) - exec_success_rate: float = Field(default=0.0, ge=0.0, le=1.0) - validation_pass_rate: float = Field(default=1.0, ge=0.0, le=1.0) - # Efficiency - total_retries: int = Field(default=0, ge=0) - avg_debug_depth: float = Field(default=0.0, ge=0.0) - plan_step_count: int = Field(default=0, ge=0) - final_step_count: int = Field(default=0, ge=0) - # Per-agent latency (ms) - analyzer_ms: int = Field(default=0, ge=0) - planner_ms: int = Field(default=0, ge=0) - coder_ms: int = Field(default=0, ge=0) - executor_ms: int = Field(default=0, ge=0) - debugger_ms: int = Field(default=0, ge=0) - verifier_ms: int = Field(default=0, ge=0) - router_ms: int = Field(default=0, ge=0) - finalizer_ms: int = Field(default=0, ge=0) - # Classification - difficulty: str = "easy" - mode: str = "live" - query_length: int = Field(default=0, ge=0) diff --git a/backend/main.py b/backend/main.py index dd543a7..9bb3af9 100644 --- a/backend/main.py +++ b/backend/main.py @@ -4,6 +4,10 @@ cross-cutting concerns (error handling) are imported from dedicated modules. """ +import asyncio +import logging +from contextlib import asynccontextmanager + from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware import os @@ -18,67 +22,75 @@ ) from middleware.error_handler import global_exception_handler from middleware.request_logger import RequestLoggerMiddleware +from api.routes import _evict_stale_sessions as _evict_sessions +_logger = logging.getLogger("uvicorn.error") -app = FastAPI( - title="Agentloop Backend", - description="Intelligent Document Processing API", - version="1.0.0" -) - -_allowed_origins_raw = os.getenv("ALLOWED_ORIGINS", "") -if not _allowed_origins_raw.strip(): - raise ValueError( - "ALLOWED_ORIGINS environment variable is not set. " - "Set it to a comma-separated list of allowed frontend origins " - "(e.g. ALLOWED_ORIGINS=https://app.example.com) before starting the server." - ) -allowed_origins = [o.strip() for o in _allowed_origins_raw.split(",") if o.strip()] - -app.add_middleware( - CORSMiddleware, - allow_origins=allowed_origins, - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -# Structured per-request logging (method, path, user, status, latency) -app.add_middleware(RequestLoggerMiddleware) +# Handle for the background eviction task — kept module-level so it can be +# inspected by health checks or tests without maintaining additional state. +_eviction_task_handle: "asyncio.Task | None" = None -app.add_exception_handler(Exception, global_exception_handler) -from api.routes import _evict_stale_sessions as _evict_sessions +# --------------------------------------------------------------------------- +# Lifespan — replaces deprecated @app.on_event("startup") / ("shutdown") +# --------------------------------------------------------------------------- -@app.on_event("startup") -async def _start_session_eviction_loop() -> None: - """B2 fix: Periodically evicts stale sessions every 60 s. +async def _eviction_loop() -> None: + """Inner loop: runs forever, catching per-tick errors without stopping.""" + while True: + await asyncio.sleep(60) + try: + _evict_sessions() + except Exception as exc: # pylint: disable=broad-except + _logger.error( + "[Eviction] Tick error (loop continues): %s", exc, exc_info=True + ) - @router.on_event(\"startup\") silently no-ops on APIRouter; registering - the task here on the FastAPI app instance ensures it actually runs. - """ - import asyncio as _asyncio - import logging as _log_mod - _l = _log_mod.getLogger("uvicorn.error") - async def _loop() -> None: - while True: - await _asyncio.sleep(60) - try: - _evict_sessions() - except Exception as exc: # pylint: disable=broad-except - _l.error("[Router] Eviction loop error: %s", exc) +async def _supervised_eviction() -> None: + """Outer supervisor: restarts the eviction loop on unexpected exit. - _asyncio.create_task(_loop()) + BUG 2 fix: Supervised background task that evicts stale sessions. + The original implementation used a bare ``asyncio.create_task`` whose loop + would die silently on any unhandled exception, causing memory to grow + unbounded. This version: -@app.on_event("startup") -async def startup_event(): + * Wraps the eviction call in a per-iteration try/except that logs + errors but never breaks the loop. + * Wraps the *entire* loop body in a supervisor that restarts the inner + task automatically after an exponentially backed-off delay (2s → 64s cap) + so a transient bug can't permanently disable eviction. + """ + _backoff = 2 # seconds — doubles on each crash, capped at 64 s + while True: + try: + _logger.info("[Eviction] Starting session-eviction loop.") + await _eviction_loop() + except asyncio.CancelledError: + _logger.info("[Eviction] Loop cancelled — shutting down cleanly.") + return # propagate cancellation — do not restart + except Exception as exc: # pylint: disable=broad-except + _logger.error( + "[Eviction] Loop crashed (%s). Restarting in %ds.", + exc, + _backoff, + exc_info=True, + ) + await asyncio.sleep(_backoff) + _backoff = min(_backoff * 2, 64) # exponential back-off, cap 64 s + else: + # _eviction_loop exited cleanly (shouldn't happen — it loops forever) + _logger.warning( + "[Eviction] Loop exited unexpectedly. Restarting in %ds.", _backoff + ) + await asyncio.sleep(_backoff) + _backoff = min(_backoff * 2, 64) + + +def _log_config_health() -> None: """Validates configuration on startup and prints config health table.""" - import logging as _logging - _log = _logging.getLogger("uvicorn.error") - checks = { "SUPABASE_URL": bool(SUPABASE_URL), "SUPABASE_PUBLISHABLE_KEY": bool(SUPABASE_PUBLISHABLE_KEY), @@ -88,27 +100,87 @@ async def startup_event(): "NVIDIA_API_KEY": bool(NVIDIA_API_KEY), } - _log.info("=" * 60) - _log.info("Agentloop config health:") + _logger.info("=" * 60) + _logger.info("Agentloop config health:") for name, ok in checks.items(): - status = "OK ✓" if ok else "MISSING / INVALID ✗" - _log.info(" %-36s %s", name, status) - _log.info("=" * 60) + label = "OK ✓" if ok else "MISSING / INVALID ✗" + _logger.info(" %-36s %s", name, label) + _logger.info("=" * 60) missing = [k for k, v in checks.items() if not v] if "SUPABASE_SERVICE_ROLE_KEY" in missing: - _log.error( + _logger.error( "SUPABASE_SERVICE_ROLE_KEY is missing or wrong format. " "It must start with 'eyJ' (JWT). " "Get it from: Supabase Dashboard → Settings → API → service_role" ) other_missing = [k for k in missing if k != "SUPABASE_SERVICE_ROLE_KEY"] if other_missing: - _log.error( + _logger.error( "Missing environment variables: %s. Check backend/.env.", ", ".join(other_missing), ) + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Application lifespan: startup and shutdown logic. + + Replaces the deprecated ``@app.on_event("startup")`` pattern. + Everything before ``yield`` runs on startup; everything after runs on + shutdown. + """ + global _eviction_task_handle # noqa: PLW0603 + + # --- Startup --- + _log_config_health() + _eviction_task_handle = asyncio.create_task(_supervised_eviction()) + + yield + + # --- Shutdown --- + if _eviction_task_handle and not _eviction_task_handle.done(): + _eviction_task_handle.cancel() + try: + await _eviction_task_handle + except asyncio.CancelledError: + pass + _logger.info("[Eviction] Background task stopped.") + + +# --------------------------------------------------------------------------- +# Application +# --------------------------------------------------------------------------- + +_allowed_origins_raw = os.getenv("ALLOWED_ORIGINS", "") +if not _allowed_origins_raw.strip(): + raise ValueError( + "ALLOWED_ORIGINS environment variable is not set. " + "Set it to a comma-separated list of allowed frontend origins " + "(e.g. ALLOWED_ORIGINS=https://app.example.com) before starting the server." + ) +allowed_origins = [o.strip() for o in _allowed_origins_raw.split(",") if o.strip()] + +app = FastAPI( + title="Agentloop Backend", + description="Intelligent Document Processing API", + version="1.0.0", + lifespan=lifespan, +) + +app.add_middleware( + CORSMiddleware, + allow_origins=allowed_origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Structured per-request logging (method, path, user, status, latency) +app.add_middleware(RequestLoggerMiddleware) + +app.add_exception_handler(Exception, global_exception_handler) + app.include_router(api_router, prefix="/api") if __name__ == "__main__": diff --git a/backend/middleware/auth.py b/backend/middleware/auth.py index 254b22f..4a5f5e1 100644 --- a/backend/middleware/auth.py +++ b/backend/middleware/auth.py @@ -5,10 +5,12 @@ authenticated user's ID and email into the route context via FastAPI's dependency injection system. -Only ES256 (asymmetric JWKS) is accepted: +Accepted algorithms: - ES256: Supabase newer projects use asymmetric keys. Public key is fetched from {SUPABASE_URL}/auth/v1/.well-known/jwks.json and cached in memory. - - HS256 fallback has been removed (algorithm confusion attack surface). + - HS256: Accepted for legacy Supabase projects and local dev where + SUPABASE_JWT_SECRET is set. Gate behind ALLOW_HS256_DEV=true in + production if you wish to restrict it to ES256 only. Audience claim ``"authenticated"`` is enforced on every token. @@ -55,46 +57,85 @@ async def my_route(auth = Depends(get_optional_user)): _JWKS_TTL_SECONDS: int = 3600 # re-fetch JWKS every hour -def _get_jwks_key(kid: str) -> Optional[Any]: +async def _refresh_jwks_cache() -> None: + """Fetches the JWKS key set from Supabase and replaces the in-process cache. + + P1-04 fix: uses ``httpx.AsyncClient`` so this never blocks the event loop. + + Raises: + RuntimeError: If SUPABASE_URL is not configured. + httpx.HTTPStatusError: If the JWKS endpoint returns a non-2xx status. + """ + global _jwks_cache, _jwks_fetched_at + + if not SUPABASE_URL: + logger.error("[Auth] SUPABASE_URL is not set — cannot fetch JWKS.") + raise RuntimeError("SUPABASE_URL is not set") + + jwks_url = f"{SUPABASE_URL.rstrip('/')}/auth/v1/.well-known/jwks.json" + async with httpx.AsyncClient(timeout=5.0) as client: + resp = await client.get(jwks_url) + resp.raise_for_status() + jwks_data = resp.json() + + new_cache: Dict[str, Any] = {} + for key_data in jwks_data.get("keys", []): + key_kid = key_data.get("kid", "") + try: + public_key = jwt.algorithms.ECAlgorithm.from_jwk(key_data) + new_cache[key_kid] = public_key + logger.info("[Auth] Loaded JWKS public key: kid=%s", key_kid) + except Exception as exc: + logger.warning("[Auth] Could not parse JWKS key kid=%s: %s", key_kid, exc) + + _jwks_cache = new_cache + _jwks_fetched_at = time.monotonic() + logger.info("[Auth] JWKS refreshed — %d key(s) loaded", len(_jwks_cache)) + + +async def _get_jwks_key(kid: str) -> Optional[Any]: """Returns the public key matching ``kid`` from Supabase's JWKS endpoint. Results are cached in-process for ``_JWKS_TTL_SECONDS`` to avoid hammering the JWKS endpoint on every request. + If the requested ``kid`` is missing from a fresh cache, a **single** + forced re-fetch is attempted to handle Supabase key rotations that + occur within the TTL window. + Args: kid: The ``kid`` (key ID) from the JWT header. Returns: A PyJWT-compatible public key object, or ``None`` if not found. """ - global _jwks_cache, _jwks_fetched_at - now = time.monotonic() + + # --- Normal TTL-based refresh --- if not _jwks_cache or (now - _jwks_fetched_at) > _JWKS_TTL_SECONDS: - if not SUPABASE_URL: - logger.error("[Auth] SUPABASE_URL is not set — cannot fetch JWKS.") - return None - jwks_url = f"{SUPABASE_URL.rstrip('/')}/auth/v1/.well-known/jwks.json" try: - resp = httpx.get(jwks_url, timeout=5.0) - resp.raise_for_status() - jwks_data = resp.json() - new_cache: Dict[str, Any] = {} - for key_data in jwks_data.get("keys", []): - key_kid = key_data.get("kid", "") - try: - public_key = jwt.algorithms.ECAlgorithm.from_jwk(key_data) - new_cache[key_kid] = public_key - logger.info("[Auth] Loaded JWKS public key: kid=%s", key_kid) - except Exception as exc: - logger.warning("[Auth] Could not parse JWKS key kid=%s: %s", key_kid, exc) - _jwks_cache = new_cache - _jwks_fetched_at = now - logger.info("[Auth] JWKS refreshed — %d key(s) loaded", len(_jwks_cache)) + await _refresh_jwks_cache() except Exception as exc: - logger.error("[Auth] Failed to fetch JWKS from %s: %s", jwks_url, exc) + logger.error("[Auth] Failed to refresh JWKS: %s", exc) # Keep stale cache on error rather than crashing + if kid in _jwks_cache: + return _jwks_cache[kid] + + # --- Retry on miss: handle key rotation within the TTL window --- + # Only re-fetch if the cache wasn't *just* refreshed (avoid tight loops). + elapsed_since_refresh = time.monotonic() - _jwks_fetched_at + if elapsed_since_refresh > 5: # guard: don't re-fetch more than once every 5 s + logger.info( + "[Auth] kid=%s not in cache (%d key(s)). " + "Forcing JWKS re-fetch for possible key rotation.", + kid, len(_jwks_cache), + ) + try: + await _refresh_jwks_cache() + except Exception as exc: + logger.error("[Auth] Forced JWKS re-fetch failed: %s", exc) + return _jwks_cache.get(kid) # --------------------------------------------------------------------------- @@ -127,12 +168,12 @@ class AuthUser(BaseModel): # --------------------------------------------------------------------------- -def _decode_supabase_jwt(token: str) -> dict: +async def _decode_supabase_jwt(token: str) -> dict: """Decodes and verifies a Supabase-issued JWT. - Accepts ONLY ES256 (asymmetric JWKS) tokens. HS256 is explicitly rejected - to eliminate algorithm confusion attacks. The audience claim is enforced - to ``"authenticated"``. + Accepts ES256 (asymmetric JWKS) and HS256 (legacy/local-dev) tokens. + HS256 requires SUPABASE_JWT_SECRET to be set. The audience claim is + enforced to ``"authenticated"``. Args: token: Raw JWT string (without the ``Bearer `` prefix). @@ -174,9 +215,9 @@ def _decode_supabase_jwt(token: str) -> dict: headers={"WWW-Authenticate": "Bearer"}, ) - # Asymmetric ES256 — resolve public key via JWKS + # Asymmetric ES256 — resolve public key via JWKS (async, P1-04 fix) kid = unverified_header.get("kid", "") - public_key = _get_jwks_key(kid) + public_key = await _get_jwks_key(kid) if public_key is None: logger.error( "[Auth] No JWKS key found for kid=%s (alg=%s). " @@ -272,7 +313,7 @@ async def get_current_user( headers={"WWW-Authenticate": "Bearer"}, ) - payload = _decode_supabase_jwt(credentials.credentials) + payload = await _decode_supabase_jwt(credentials.credentials) auth_user = _extract_auth_user(payload) logger.debug("[Auth] Authenticated user: %s", auth_user.user_id) return auth_user @@ -297,7 +338,7 @@ async def get_optional_user( return None try: - payload = _decode_supabase_jwt(credentials.credentials) + payload = await _decode_supabase_jwt(credentials.credentials) return _extract_auth_user(payload) except HTTPException: return None diff --git a/backend/models/metrics_schema.py b/backend/models/metrics_schema.py deleted file mode 100644 index 8ad93ed..0000000 --- a/backend/models/metrics_schema.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Pydantic schemas for DS-STAR evaluation metrics. - -Captures per-round timing, token usage, and task complexity tagging to -mirror the ablation study methodology in the DS-STAR paper (easy vs. hard -task convergence: 3.0 vs 5.6 average rounds on DABStep). -""" - -from typing import Dict, List, Literal, Optional - -from pydantic import BaseModel, Field - - -class RoundMetric(BaseModel): - """Timing and status data captured for a single agent loop round. - - Attributes: - round_num: 1-based round number within the run. - planner_ms: Milliseconds spent in the PlannerAgent call (round 1 only). - coder_ms: Milliseconds spent in the CoderAgent call. - executor_ms: Milliseconds spent in subprocess code execution. - verifier_ms: Milliseconds spent in the VerifierAgent call. - router_ms: Milliseconds spent in the RouterAgent call (0 if not triggered). - total_ms: Total wall-clock milliseconds for this round. - is_sufficient: Whether VerifierAgent approved this round's output. - verifier_confidence: Confidence score emitted by the verifier (0.0–1.0). - exec_success: Whether the executor subprocess exited with code 0. - """ - - round_num: int = Field(ge=1, description="1-based round index.") - planner_ms: int = Field(default=0, ge=0, description="Planner latency (ms).") - coder_ms: int = Field(default=0, ge=0, description="Coder latency (ms).") - executor_ms: int = Field(default=0, ge=0, description="Executor latency (ms).") - verifier_ms: int = Field(default=0, ge=0, description="Verifier latency (ms).") - router_ms: int = Field(default=0, ge=0, description="Router latency (ms).") - total_ms: int = Field(default=0, ge=0, description="Total round wall-clock (ms).") - prompt_tokens: int = Field(default=0, ge=0, description="Total prompt tokens in this round.") - completion_tokens: int = Field(default=0, ge=0, description="Total completion tokens in this round.") - is_sufficient: bool = Field(default=False) - verifier_confidence: float = Field(default=0.0, ge=0.0, le=1.0) - exec_success: bool = Field(default=False) - - -class RunMetrics(BaseModel): - """Full evaluation-quality metrics for a single DS-STAR agent run. - - These metrics enable tracking the convergence patterns described in the - DS-STAR paper (Section 4.2 ablations), differentiating easy vs. hard tasks. - - Attributes: - run_id: UUID matching the agent_runs Supabase row. - query: Original user query text. - complexity: Task complexity tag — 'easy' (single file) or 'hard' (multi-file). - file_count: Number of distinct files in the processing context. - rounds_completed: Total refinement rounds consumed. - rounds_until_sufficient: Round index at which Verifier first approved output - (equals rounds_completed if max rounds hit without approval). - total_run_ms: Total wall-clock duration of the entire agent run (ms). - final_status: "completed" | "failed" | "max_rounds_reached". - rounds: Ordered per-round metric breakdown. - """ - - run_id: str - query: str - complexity: Literal["easy", "hard"] = Field( - description="'easy' = single file, 'hard' = 2+ files (per DABStep taxonomy)." - ) - file_count: int = Field(default=1, ge=0) - rounds_completed: int = Field(default=0, ge=0) - rounds_until_sufficient: int = Field(default=0, ge=0) - total_run_ms: int = Field(default=0, ge=0) - final_status: Literal["completed", "failed", "max_rounds_reached"] = "completed" - rounds: List[RoundMetric] = Field(default_factory=list) - - def summary(self) -> Dict: - """Returns a flat serialisable dict suitable for Supabase jsonb column. - - Returns: - Dict with top-level metrics and per-round breakdown. - """ - return { - "complexity": self.complexity, - "file_count": self.file_count, - "rounds_completed": self.rounds_completed, - "rounds_until_sufficient": self.rounds_until_sufficient, - "total_run_ms": self.total_run_ms, - "final_status": self.final_status, - "per_round": [r.model_dump() for r in self.rounds], - } - - -class RoundTimingCollector: - """Helper for recording per-stage timing within a single agent round. - - Usage:: - - collector = RoundTimingCollector(round_num=1) - collector.record("coder", elapsed_ms) - metric = collector.build() - """ - - def __init__(self, round_num: int) -> None: - self._round_num = round_num - self._timings: Dict[str, int] = {} - self._prompt_tokens = 0 - self._completion_tokens = 0 - - def record(self, stage: str, elapsed_ms: int) -> None: - """Manually records a timing entry. - - Args: - stage: Stage name (e.g. ``"coder"``, ``"executor"``). - elapsed_ms: Elapsed milliseconds for the stage. - """ - self._timings[stage] = elapsed_ms - - def get(self, stage: str) -> int: - """Returns the recorded timing for a stage, or 0 if not recorded. - - Args: - stage: Stage name. - - Returns: - Recorded elapsed milliseconds, or 0. - """ - return self._timings.get(stage, 0) - - def record_tokens(self, prompt: int, completion: int) -> None: - """Aggregates token usage for the round.""" - self._prompt_tokens += prompt - self._completion_tokens += completion - - def build( - self, - is_sufficient: bool = False, - verifier_confidence: float = 0.0, - exec_success: bool = False, - ) -> RoundMetric: - """Assembles a ``RoundMetric`` from recorded timings and tokens.""" - total = sum(self._timings.values()) - return RoundMetric( - round_num=self._round_num, - planner_ms=self._timings.get("planner", 0), - coder_ms=self._timings.get("coder", 0), - executor_ms=self._timings.get("executor", 0), - verifier_ms=self._timings.get("verifier", 0), - router_ms=self._timings.get("router", 0), - total_ms=total, - prompt_tokens=self._prompt_tokens, - completion_tokens=self._completion_tokens, - is_sufficient=is_sufficient, - verifier_confidence=verifier_confidence, - exec_success=exec_success, - ) diff --git a/backend/models/multi_file_context.py b/backend/models/multi_file_context.py new file mode 100644 index 0000000..b9e8042 --- /dev/null +++ b/backend/models/multi_file_context.py @@ -0,0 +1,247 @@ +"""Multi-file context models for the workspace join feature. + +Defines the canonical Pydantic models used throughout the multi-file +pipeline: + + ColumnMeta — one column from a parsed file schema + FileSchema — full schema representation of one uploaded file + JoinCandidate — a probable join key pair between two files + MultiFileContext — container passed between Analyzer → Planner → Coder + +The ``to_prompt_str()`` method on MultiFileContext produces a compact, +LLM-readable summary under 600 tokens regardless of file count. It is +injected into agent prompts as the authoritative multi-file schema context. +""" + +from __future__ import annotations + +import textwrap +from typing import List + +from pydantic import BaseModel, Field + + +# --------------------------------------------------------------------------- +# Column-level metadata +# --------------------------------------------------------------------------- + +class ColumnMeta(BaseModel): + """Metadata for a single column in a parsed file.""" + + name: str = Field(description="Column name as it appears in the file header.") + dtype: str = Field(description="Pandas dtype string, e.g. 'int64', 'object', 'float64'.") + sample_values: List[str] = Field( + default_factory=list, + description="Up to 5 representative non-null values serialised as strings.", + ) + + +# --------------------------------------------------------------------------- +# Per-file schema +# --------------------------------------------------------------------------- + +class FileSchema(BaseModel): + """Full schema representation of one uploaded workspace file.""" + + var_name: str = Field( + description="Python variable name assigned to this file, e.g. 'df1', 'df2'." + ) + file_name: str = Field(description="Original filename, e.g. 'orders.csv'.") + file_path: str = Field(description="Absolute path to the file on disk.") + file_type: str = Field(description="One of: csv, xlsx, parquet, md, json.") + row_count: int = Field(default=0, description="Number of rows in the file.") + columns: List[ColumnMeta] = Field( + default_factory=list, + description="Column schemas for this file.", + ) + + +# --------------------------------------------------------------------------- +# Join candidate +# --------------------------------------------------------------------------- + +class JoinCandidate(BaseModel): + """A probable join key relationship between two files.""" + + left_var: str = Field(description="Variable name of the left-hand file, e.g. 'df1'.") + right_var: str = Field(description="Variable name of the right-hand file, e.g. 'df2'.") + left_col: str = Field(description="Column name in the left file.") + right_col: str = Field(description="Column name in the right file.") + confidence: float = Field( + description="Confidence score in [0.0, 1.0]. Higher = more likely a valid join key.", + ge=0.0, + le=1.0, + ) + reason: str = Field( + description="Human-readable explanation of why this candidate was selected." + ) + + +# --------------------------------------------------------------------------- +# Top-level container +# --------------------------------------------------------------------------- + +# Maximum columns to include per file in the prompt string before truncating. +_MAX_COLS_PER_FILE = 8 +# Maximum sample values to show per column in the prompt string. +_MAX_SAMPLES_PER_COL = 3 + + +def _score_column_for_prompt(col: ColumnMeta) -> float: + """Heuristic priority score for selecting columns shown in to_prompt_str(). + + Numeric columns and low-cardinality strings are most likely to be join + keys or analysis targets, so they rank highest. + + Args: + col: ColumnMeta to score. + + Returns: + Float score — higher is more important to show. + """ + dtype_lower = col.dtype.lower() + is_numeric = any(k in dtype_lower for k in ("int", "float", "decimal", "numeric")) + is_object = "object" in dtype_lower or "string" in dtype_lower + unique_samples = len(set(col.sample_values)) + # Low cardinality = likely a key or categorical column + is_low_cardinality = is_object and unique_samples <= 5 + return ( + 3.0 if is_numeric else 2.0 if is_low_cardinality else 1.0 + ) + + +class MultiFileContext(BaseModel): + """Container for all per-file schemas and inferred join candidates. + + Passed between the schema_merger, Analyzer, Planner, and Coder so that + every agent stage has the same ground-truth multi-file view. + """ + + files: List[FileSchema] = Field( + default_factory=list, + description="One FileSchema per uploaded file, ordered by upload_order.", + ) + join_candidates: List[JoinCandidate] = Field( + default_factory=list, + description="Inferred join key pairs, sorted by confidence descending.", + ) + + def to_prompt_str(self) -> str: + """Compact, LLM-readable summary of the multi-file context. + + Output format:: + + FILES: + df1 = orders.csv (12450 rows) | columns: order_id(int64), customer_id(int64), amount(float64), ... + df2 = customers.csv (3200 rows) | columns: cust_id(int64), name(object), region(object), ... + JOIN CANDIDATES (by confidence): + df1['customer_id'] <-> df2['cust_id'] [0.95 — exact name match after normalisation] + df1['region_code'] <-> df2['region'] [0.60 — fuzzy name match (ratio: 0.83)] + + Truncation rules (to stay under 600 tokens): + - At most ``_MAX_COLS_PER_FILE`` columns per file (highest-priority first). + - At most ``_MAX_SAMPLES_PER_COL`` sample values per column. + - Sample values truncated to 20 characters each. + + Returns: + Formatted multi-line string safe to inject directly into any + LLM prompt as a context block. + """ + lines: List[str] = ["FILES:"] + + for fs in self.files: + # Select top columns by priority score + ranked = sorted(fs.columns, key=_score_column_for_prompt, reverse=True) + shown = ranked[:_MAX_COLS_PER_FILE] + truncated = len(fs.columns) - len(shown) + + col_parts: List[str] = [] + for col in shown: + samples = col.sample_values[:_MAX_SAMPLES_PER_COL] + samples_str = ", ".join(str(v)[:20] for v in samples) + samples_display = f" [{samples_str}]" if samples_str else "" + col_parts.append(f"{col.name}({col.dtype}){samples_display}") + + col_str = ", ".join(col_parts) + if truncated > 0: + col_str += f", ...+{truncated} more" + + lines.append( + f" {fs.var_name} = {fs.file_name} ({fs.row_count:,} rows)" + f" | columns: {col_str}" + ) + + if self.join_candidates: + lines.append("JOIN CANDIDATES (by confidence):") + for jc in self.join_candidates: + lines.append( + f" {jc.left_var}['{jc.left_col}'] <-> " + f"{jc.right_var}['{jc.right_col}']" + f" [{jc.confidence:.2f} — {jc.reason}]" + ) + else: + lines.append("JOIN CANDIDATES: (none detected)") + + return "\n".join(lines) + + def to_reader_header(self, workspace_id: str) -> str: + """Generates the Python data-loading header for the CoderAgent. + + Produces one ``pd.read_*`` call per file, using the correct reader + for each file_type. The generated variable names match ``var_name`` + fields exactly so the coder's subsequent code can reference them. + + Args: + workspace_id: Workspace UUID used to resolve the storage path. + + Returns: + Multi-line Python source string that pre-loads all DataFrames. + """ + reader_map = { + "csv": "pd.read_csv", + "xlsx": "pd.read_excel", + "parquet": "pd.read_parquet", + "json": "pd.read_json", + "md": None, # handled specially below + } + + lines = ["import pandas as pd", "import os", ""] + + for fs in self.files: + path_expr = repr(fs.file_path) + ft = fs.file_type.lower() + reader = reader_map.get(ft) + + if reader is None: + # Markdown: extract first table using regex fallback + lines.append( + textwrap.dedent(f"""\ + # Load markdown file and extract the first table + _md_path_{fs.var_name} = {path_expr} + _md_lines = open(_md_path_{fs.var_name}, encoding='utf-8', errors='replace').readlines() + _md_table_lines = [l for l in _md_lines if l.strip().startswith('|')] + if _md_table_lines: + import io as _io + _md_csv = '\\n'.join( + ','.join(c.strip() for c in row.strip().strip('|').split('|')) + for row in _md_table_lines + if not set(row.replace('|', '').replace('-', '').replace(' ', '')) == set() + ) + {fs.var_name} = pd.read_csv(_io.StringIO(_md_csv)) + else: + {fs.var_name} = pd.DataFrame() + print('WARNING: No markdown table found in {fs.file_name}') + """) + ) + else: + lines.append(f"{fs.var_name} = {reader}({path_expr})") + + lines.append("") + # Verification prints required by the VerifierAgent + for fs in self.files: + lines.append( + f"print(f'{fs.var_name} shape: {{{fs.var_name}.shape}}')" + ) + lines.append("") + + return "\n".join(lines) diff --git a/backend/requirements.txt b/backend/requirements.txt index 660b505..2e1522e 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -15,6 +15,9 @@ tenacity>=8.3.0 sse-starlette>=2.1.0 httpx>=0.27.0 +# Plotly static image export (PNG/JPEG) for agent-generated charts +kaleido>=0.2.1 + # NVIDIA NIM (replaces google-generativeai) langchain-nvidia-ai-endpoints>=0.3.0 diff --git a/backend/services/parsers/__init__.py b/backend/services/parsers/__init__.py index 0314f66..20801fc 100644 --- a/backend/services/parsers/__init__.py +++ b/backend/services/parsers/__init__.py @@ -1 +1,67 @@ -"""Parsers sub-package under services.""" +"""Parser package — exposes ParseWarning for callers to handle near-empty results. + +BUG 4 fix: Parsers now raise ParseWarning (a subclass of UserWarning, not +Exception) when they produce content that is syntactically valid but so sparse +that passing it to the LLM would likely cause hallucinations. This keeps the +normal ValueError path for hard failures and adds a separate, named warning +path for graceful degradation. +""" + + +import math +from typing import Any, Dict, List + +import pandas as pd + + +def sanitize_records(df: pd.DataFrame, n: int = 5) -> List[Dict[str, Any]]: + """Converts a DataFrame head to a list of dicts with NaN/Inf replaced by None. + + The JSON specification does not support ``NaN``, ``Infinity``, or + ``-Infinity``. Supabase (and ``json.dumps`` in strict mode) will + reject any payload containing these sentinel values. This helper + ensures all parser outputs are JSON-safe before they reach the DB + insert layer. + + Args: + df: Source DataFrame. + n: Number of head rows to include. + + Returns: + List of record dicts safe for ``json.dumps()``. + """ + records = df.head(n).to_dict(orient="records") + return _sanitize_value(records) + + +def _sanitize_value(obj: Any) -> Any: + """Recursively replace non-JSON-compliant floats with None.""" + if isinstance(obj, float): + if math.isnan(obj) or math.isinf(obj): + return None + return obj + if isinstance(obj, dict): + return {k: _sanitize_value(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_sanitize_value(item) for item in obj] + return obj + + +class ParseWarning(UserWarning): + """Raised when a parser succeeds but yields near-empty or unusable content. + + Propagated by process_service.process_documents() as an ``warnings`` + warning, caught at the API layer, and emitted as an SSE ``"warning"`` event + so the user knows their file produced minimal context *before* the agent + loop begins. + + Args: + message: Human-readable description of why the content is insufficient. + filename: The file that triggered the warning. + char_count: Number of meaningful characters actually extracted. + """ + + def __init__(self, message: str, filename: str = "", char_count: int = 0) -> None: + super().__init__(message) + self.filename = filename + self.char_count = char_count diff --git a/backend/services/parsers/csv_parser.py b/backend/services/parsers/csv_parser.py index 68530ca..d96f702 100644 --- a/backend/services/parsers/csv_parser.py +++ b/backend/services/parsers/csv_parser.py @@ -13,6 +13,7 @@ import pandas as pd from core.validation import sanitize_text +from services.parsers import sanitize_records def parse_csv(file_name: str, file_content: bytes) -> Dict[str, Any]: @@ -38,6 +39,10 @@ def parse_csv(file_name: str, file_content: bytes) -> Dict[str, Any]: # Serialize dtypes as a readable dict (str keys, str values) dtypes_map = {col: str(dtype) for col, dtype in df.dtypes.items()} + # sanitize_records replaces NaN/Inf with None so the result is + # JSON-serializable (required for Supabase schema_json column). + safe_sample = sanitize_records(df, n=5) + return { "file_name": file_name, "source_type": "csv", @@ -48,10 +53,11 @@ def parse_csv(file_name: str, file_content: bytes) -> Dict[str, Any]: "dtypes": dtypes_map, "shape": list(df.shape), # [rows, cols] "row_count": len(df), - "sample_rows": df.head(5).to_dict(orient="records"), + "sample_rows": safe_sample, # Legacy key kept for any downstream consumers - "sample_data": df.head(5).to_dict(orient="records"), + "sample_data": safe_sample, }, } except Exception as exc: raise ValueError(f"Failed to parse CSV: {exc}") from exc + diff --git a/backend/services/parsers/excel_parser.py b/backend/services/parsers/excel_parser.py index cdccf78..637f2b7 100644 --- a/backend/services/parsers/excel_parser.py +++ b/backend/services/parsers/excel_parser.py @@ -14,6 +14,7 @@ import pandas as pd from core.validation import sanitize_text +from services.parsers import sanitize_records def parse_excel(file_name: str, file_content: bytes) -> Dict[str, Any]: @@ -59,12 +60,12 @@ def parse_excel(file_name: str, file_content: bytes) -> Dict[str, Any]: "dtypes": dtypes_map, "shape": list(first_df.shape), # [rows, cols] "row_count": len(first_df), - "sample_rows": first_df.head(5).to_dict(orient="records"), + "sample_rows": sanitize_records(first_df, n=5), # XLSX-specific extras "sheet_count": len(sheets), "sheet_names": sheets, # Legacy key kept for any downstream consumers - "preview_first_sheet": first_df.head(5).to_dict(orient="records"), + "preview_first_sheet": sanitize_records(first_df, n=5), }, } except Exception as exc: diff --git a/backend/services/parsers/md_parser.py b/backend/services/parsers/md_parser.py index 5f0e7e0..1192810 100644 --- a/backend/services/parsers/md_parser.py +++ b/backend/services/parsers/md_parser.py @@ -2,11 +2,20 @@ Extracts and parses standard Markdown text files into unified context, mapping the logic via the `markdown` module for extended functionality later. + +BUG 4 fix: Raises ParseWarning when the extracted text is near-empty (< 50 +characters), allowing process_service to surface the issue as an SSE warning +before the agent loop starts rather than silently passing empty context. """ from typing import Dict, Any import markdown + from core.validation import sanitize_text +from services.parsers import ParseWarning + +# Minimum meaningful character count before we emit a ParseWarning +_MIN_CONTENT_CHARS: int = 50 def parse_md(file_name: str, file_content: bytes) -> Dict[str, Any]: @@ -20,23 +29,38 @@ def parse_md(file_name: str, file_content: bytes) -> Dict[str, Any]: Dict[str, Any]: The extracted text formatted unified. Raises: - ValueError: If the file cannot be read. + ValueError: If the file cannot be decoded or the markdown library fails. + ParseWarning: If the extracted content is shorter than _MIN_CONTENT_CHARS, + indicating the file is likely empty or contains only whitespace/markup. """ try: content = file_content.decode('utf-8') + except UnicodeDecodeError as exc: + raise ValueError(f"Failed to decode Markdown as UTF-8: {exc}") from exc + try: # Parse into HTML to strip raw structural blocks, then pass to sanitizer html_converted = markdown.markdown(content) sanitized = sanitize_text(html_converted) + except Exception as exc: # pylint: disable=broad-except + raise ValueError(f"Failed to parse Markdown: {exc}") from exc + + if len(sanitized.strip()) < _MIN_CONTENT_CHARS: + raise ParseWarning( + f"'{file_name}' produced near-empty content after Markdown parsing " + f"({len(sanitized.strip())} chars). The file may be blank or contain " + "only structural markup. Agent context will be minimal.", + filename=file_name, + char_count=len(sanitized.strip()), + ) - return { - "file_name": file_name, - "source_type": "md", - "sanitized_content": sanitized[:5000], - "metadata": { - "char_count": len(sanitized), - "content_preview": sanitized[:1000] - } + return { + "file_name": file_name, + "source_type": "md", + "sanitized_content": sanitized[:5000], + "metadata": { + "char_count": len(sanitized), + "content_preview": sanitized[:1000] } - except Exception as e: - raise ValueError(f"Failed to parse Markdown: {e}") + } + diff --git a/backend/services/parsers/parquet_parser.py b/backend/services/parsers/parquet_parser.py index 608c280..7d3ea81 100644 --- a/backend/services/parsers/parquet_parser.py +++ b/backend/services/parsers/parquet_parser.py @@ -10,6 +10,7 @@ import io from core.validation import sanitize_text +from services.parsers import sanitize_records def parse_parquet(file_name: str, file_content: bytes) -> Dict[str, Any]: @@ -44,7 +45,7 @@ def parse_parquet(file_name: str, file_content: bytes) -> Dict[str, Any]: "dtypes": dtypes_map, "shape": list(df.shape), "row_count": len(df), - "sample_rows": df.head(5).to_dict(orient="records"), + "sample_rows": sanitize_records(df, n=5), }, } except ImportError as exc: diff --git a/backend/services/parsers/pdf_parser.py b/backend/services/parsers/pdf_parser.py index ef03726..b346872 100644 --- a/backend/services/parsers/pdf_parser.py +++ b/backend/services/parsers/pdf_parser.py @@ -6,6 +6,10 @@ by FileAnalyzerAgent. Also changed from a hard 10-page cap to a character- budget approach (all pages, up to 20 000 chars) so long technical documents aren't arbitrarily truncated at page 10. + +BUG 4 fix: Raises ParseWarning when the extracted text is near-empty, which +typically happens with scan-only PDFs or encrypted documents, preventing +silent empty-context hallucinations in the agent loop. """ import io @@ -14,9 +18,12 @@ import pypdf from core.validation import sanitize_text +from services.parsers import ParseWarning # Maximum characters to extract across all pages _MAX_CHARS: int = 20_000 +# Minimum meaningful characters before a ParseWarning is raised +_MIN_CONTENT_CHARS: int = 50 def parse_pdf(file_name: str, file_content: bytes) -> Dict[str, Any]: @@ -30,7 +37,9 @@ def parse_pdf(file_name: str, file_content: bytes) -> Dict[str, Any]: Dict[str, Any]: Structured unified document holding text blocks. Raises: - ValueError: If the file is unreadable. + ValueError: If the file is unreadable (corrupt, wrong format). + ParseWarning: If the extracted text is shorter than _MIN_CONTENT_CHARS, + indicating a scan-only or encrypted PDF where text extraction failed. """ try: content_accumulator = [] @@ -53,14 +62,28 @@ def parse_pdf(file_name: str, file_content: bytes) -> Dict[str, Any]: full_text = "\n".join(content_accumulator) sanitized = sanitize_text(full_text) - return { - "file_name": file_name, - "source_type": "pdf", - "sanitized_content": sanitized[:8000], # higher cap than original 5000 - "metadata": { - "pages": page_count, # matches analyzer L61 ("pages") - "text_preview": sanitized[:1000], - }, - } - except Exception as exc: + except pypdf.errors.PdfReadError as exc: + raise ValueError(f"PDF is corrupt or cannot be read: {exc}") from exc + except Exception as exc: # pylint: disable=broad-except raise ValueError(f"Failed to parse PDF via pypdf: {exc}") from exc + + if len(sanitized.strip()) < _MIN_CONTENT_CHARS: + raise ParseWarning( + f"'{file_name}' yielded near-empty text after PDF extraction " + f"({len(sanitized.strip())} chars across {page_count} page(s)). " + "The PDF may be scan-only (image-based) or password-protected. " + "Agent context will be minimal.", + filename=file_name, + char_count=len(sanitized.strip()), + ) + + return { + "file_name": file_name, + "source_type": "pdf", + "sanitized_content": sanitized[:8000], # higher cap than original 5000 + "metadata": { + "pages": page_count, # matches analyzer L61 ("pages") + "text_preview": sanitized[:1000], + }, + } + diff --git a/backend/services/parsers/txt_parser.py b/backend/services/parsers/txt_parser.py index bd1d895..490c84a 100644 --- a/backend/services/parsers/txt_parser.py +++ b/backend/services/parsers/txt_parser.py @@ -2,10 +2,18 @@ Reads standard text files and returns raw blocks filtered through the unified document context schemas and sanitizers. + +BUG 4 fix: Raises ParseWarning when the extracted text is near-empty (< 50 +characters), allowing process_service to surface the issue as an SSE warning +before the agent loop starts rather than silently passing empty context. """ from typing import Dict, Any + from core.validation import sanitize_text +from services.parsers import ParseWarning + +_MIN_CONTENT_CHARS: int = 50 def parse_txt(file_name: str, file_content: bytes) -> Dict[str, Any]: @@ -21,20 +29,34 @@ def parse_txt(file_name: str, file_content: bytes) -> Dict[str, Any]: Raises: ValueError: If string decode format is misaligned. + ParseWarning: If the content is shorter than _MIN_CONTENT_CHARS. """ try: content = file_content.decode('utf-8') + except UnicodeDecodeError as exc: + raise ValueError(f"Failed to decode TXT file as UTF-8: {exc}") from exc + try: sanitized = sanitize_text(content) - - return { - "file_name": file_name, - "source_type": "txt", - "sanitized_content": sanitized[:5000], - "metadata": { - "char_count": len(sanitized), - "preview": sanitized[:100] + "..." if len(sanitized) > 100 else sanitized - } + except Exception as exc: # pylint: disable=broad-except + raise ValueError(f"Failed to sanitize TXT content: {exc}") from exc + + if len(sanitized.strip()) < _MIN_CONTENT_CHARS: + raise ParseWarning( + f"'{file_name}' produced near-empty content after parsing " + f"({len(sanitized.strip())} chars). The file may be blank. " + "Agent context will be minimal.", + filename=file_name, + char_count=len(sanitized.strip()), + ) + + return { + "file_name": file_name, + "source_type": "txt", + "sanitized_content": sanitized[:5000], + "metadata": { + "char_count": len(sanitized), + "preview": sanitized[:100] + "..." if len(sanitized) > 100 else sanitized } - except Exception as e: - raise ValueError(f"Failed to parse TXT: {e}") + } + diff --git a/backend/services/process_service.py b/backend/services/process_service.py index 326e2d9..9844092 100644 --- a/backend/services/process_service.py +++ b/backend/services/process_service.py @@ -4,10 +4,20 @@ aggregating output into a normalised schema map matching UnifiedDocumentContext. Added: Parquet (.parquet) format support. + +BUG 4 fix: process_documents() now distinguishes between hard parse failures +(ValueError → stored as an error entry) and near-empty parse warnings +(ParseWarning → stored as a successful entry *plus* a warning message in the +returned ``parse_warnings`` list). The API layer is responsible for emitting +each warning as an SSE "warning" event before the agent loop begins, ensuring +users know their file produced minimal context rather than silently receiving +hallucinated column names from the LLM. """ + from typing import Any, Dict, List +from services.parsers import ParseWarning from services.parsers.csv_parser import parse_csv from services.parsers.txt_parser import parse_txt from services.parsers.excel_parser import parse_excel @@ -37,6 +47,13 @@ def process_documents( ) -> Dict[str, Any]: """Routes files from the session cache to parsing logic. + BUG 4 fix: ParseWarning from individual parsers is captured per-file and + stored in ``aggregated_context["parse_warnings"]``. Hard failures + (ValueError and unexpected exceptions) continue to be stored as error + entries in ``combined_extractions``. Callers (the agent controller / + research controller) must iterate ``parse_warnings`` and emit each message + as an SSE ``"warning"`` event before starting the orchestrator. + Args: file_names: Filenames matching keys in the session's file cache. session_id: Session identifier used to look up files from the @@ -44,11 +61,14 @@ def process_documents( Returns: Dict[str, Any]: A unified memory map of normalised contextual documents, - guaranteed to contain no nan/inf float values. + guaranteed to contain no nan/inf float values, plus a ``parse_warnings`` + list of human-readable warning strings (may be empty). """ aggregated_context: Dict[str, Any] = { "files_processed": 0, "combined_extractions": {}, + # BUG 4: new key — list of (filename, message) tuples for near-empty results + "parse_warnings": [], } parser_map = { @@ -78,16 +98,50 @@ def process_documents( if parser_fn: try: + # BUG 4: Use warnings.catch_warnings to intercept ParseWarning + # without breaking the normal parse path. ParseWarning is + # raised (not warned) by parsers, so we catch it explicitly. extraction = parser_fn(filename, file_content) aggregated_context["combined_extractions"][filename] = extraction aggregated_context["files_processed"] += 1 - except Exception as exc: # pylint: disable=broad-except + + except ParseWarning as pw: + # Near-empty result: store the best-effort extraction we can + # reconstruct (empty sanitized_content) AND log the warning so + # the caller can surface it before the agent loop starts. + aggregated_context["combined_extractions"][filename] = { + "file_name": filename, + "source_type": ext, + "sanitized_content": "", + "metadata": { + "warning": str(pw), + "char_count": getattr(pw, "char_count", 0), + }, + } + aggregated_context["parse_warnings"].append({ + "filename": filename, + "message": str(pw), + }) + # Count as processed — the file *was* readable, just sparse + aggregated_context["files_processed"] += 1 + + except ValueError as exc: + # Hard parse failure — file is unreadable or corrupt aggregated_context["combined_extractions"][filename] = { "file_name": filename, "source_type": ext, "sanitized_content": "", "metadata": {"error": f"Parse failure: {str(exc)}"}, } + + except Exception as exc: # pylint: disable=broad-except + # Unexpected error — treat as hard failure + aggregated_context["combined_extractions"][filename] = { + "file_name": filename, + "source_type": ext, + "sanitized_content": "", + "metadata": {"error": f"Unexpected parse error: {str(exc)}"}, + } else: aggregated_context["combined_extractions"][filename] = { "file_name": filename, diff --git a/backend/services/schema_merger.py b/backend/services/schema_merger.py new file mode 100644 index 0000000..114b37e --- /dev/null +++ b/backend/services/schema_merger.py @@ -0,0 +1,356 @@ +"""Schema merger service — infers join candidates from stored file metadata. + +Public API +---------- + async def merge_schemas(file_metas: List[dict]) -> MultiFileContext + +``file_metas`` is a list of dicts matching the ``workspace_files`` table row +shape (file_name, file_path, file_type, row_count, schema_json, upload_order). + +The function assigns variable names df1…dfN by ``upload_order`` (ascending), +builds ``FileSchema`` objects from the stored ``schema_json``, then runs three +join-inference passes over every pair of files (i < j): + + Pass 1 — Exact normalised name match → confidence 0.95 + Pass 2 — Same dtype + sample value overlap >30% → confidence 0.70 + Pass 3 — difflib fuzzy ratio > 0.80 → confidence 0.55 + +After all passes the candidate list is deduplicated (highest confidence wins), +sorted descending, and capped at 5 entries. + +No file I/O is performed — all inference is done from ``schema_json`` alone. +""" + +from __future__ import annotations + +import difflib +import logging +import re +from typing import Any, Dict, List, Optional, Tuple + +from models.multi_file_context import ( + ColumnMeta, + FileSchema, + JoinCandidate, + MultiFileContext, +) + +logger = logging.getLogger("uvicorn.info") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_MAX_CANDIDATES = 5 +_FUZZY_RATIO_THRESHOLD = 0.80 +_SAMPLE_OVERLAP_THRESHOLD = 0.30 + +_CONFIDENCE_EXACT = 0.95 +_CONFIDENCE_SAMPLE = 0.70 +_CONFIDENCE_FUZZY = 0.55 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _normalise_col(name: str) -> str: + """Normalises a column name for exact-match comparison. + + Steps: + 1. Lowercase + 2. Strip leading/trailing whitespace + 3. Replace spaces and hyphens with underscores + + Args: + name: Raw column name from the schema. + + Returns: + Normalised column name string. + """ + return re.sub(r"[\s\-]+", "_", name.strip().lower()) + + +def _build_file_schema(meta: Dict[str, Any], var_name: str) -> FileSchema: + """Constructs a FileSchema from a workspace_files row dict. + + The ``schema_json`` field stores the parser's ``metadata`` dict which + follows the ``UnifiedDocumentContext`` shape:: + + { + "columns": ["col_a", "col_b", ...], + "dtypes": {"col_a": "int64", "col_b": "object", ...}, + "sample_rows": [{"col_a": 1, "col_b": "x"}, ...], + ... + } + + Args: + meta: A workspace_files table row dict. + var_name: Python variable name to assign (e.g. 'df1'). + + Returns: + Populated FileSchema instance. + """ + schema_json: Dict[str, Any] = meta.get("schema_json") or {} + + columns_list: List[str] = schema_json.get("columns") or [] + dtypes_map: Dict[str, str] = schema_json.get("dtypes") or {} + sample_rows: List[Dict[str, Any]] = schema_json.get("sample_rows") or [] + + col_metas: List[ColumnMeta] = [] + for col_name in columns_list: + dtype = dtypes_map.get(col_name, "object") + # Collect up to 5 unique non-null sample values for this column + samples: List[str] = [] + seen: set = set() + for row in sample_rows: + val = row.get(col_name) + if val is None: + continue + val_str = str(val)[:50] + if val_str not in seen: + seen.add(val_str) + samples.append(val_str) + if len(samples) >= 5: + break + + col_metas.append(ColumnMeta(name=col_name, dtype=dtype, sample_values=samples)) + + return FileSchema( + var_name=var_name, + file_name=meta.get("file_name", "unknown"), + file_path=meta.get("file_path", ""), + file_type=meta.get("file_type", "csv"), + row_count=meta.get("row_count") or 0, + columns=col_metas, + ) + + +def _candidate_key(left_col: str, right_col: str, left_var: str, right_var: str) -> str: + """Returns a stable deduplication key for a join candidate.""" + return f"{left_var}.{left_col}:{right_var}.{right_col}" + + +# --------------------------------------------------------------------------- +# The three inference passes +# --------------------------------------------------------------------------- + +def _pass1_exact( + fs_i: FileSchema, + fs_j: FileSchema, +) -> List[JoinCandidate]: + """Pass 1: Exact normalised name match. + + Args: + fs_i: Left file schema. + fs_j: Right file schema. + + Returns: + List of JoinCandidates with confidence=0.95. + """ + candidates: List[JoinCandidate] = [] + norm_j = {_normalise_col(c.name): c.name for c in fs_j.columns} + + for col_i in fs_i.columns: + norm_i = _normalise_col(col_i.name) + if norm_i in norm_j: + candidates.append( + JoinCandidate( + left_var=fs_i.var_name, + right_var=fs_j.var_name, + left_col=col_i.name, + right_col=norm_j[norm_i], + confidence=_CONFIDENCE_EXACT, + reason="exact name match after normalisation", + ) + ) + return candidates + + +def _pass2_sample_overlap( + fs_i: FileSchema, + fs_j: FileSchema, +) -> List[JoinCandidate]: + """Pass 2: Matching dtype + sample value set intersection > 30%. + + Args: + fs_i: Left file schema. + fs_j: Right file schema. + + Returns: + List of JoinCandidates with confidence=0.70. + """ + candidates: List[JoinCandidate] = [] + + for col_i in fs_i.columns: + if not col_i.sample_values: + continue + set_i = set(col_i.sample_values) + + for col_j in fs_j.columns: + if not col_j.sample_values: + continue + # Dtypes must match (normalised: strip the bit-width suffix for int/float comparison) + dtype_i = col_i.dtype.lower().rstrip("0123456789") + dtype_j = col_j.dtype.lower().rstrip("0123456789") + if dtype_i != dtype_j: + continue + + set_j = set(col_j.sample_values) + intersection = set_i & set_j + overlap_ratio = len(intersection) / min(len(set_i), len(set_j)) + + if overlap_ratio > _SAMPLE_OVERLAP_THRESHOLD: + pct = int(overlap_ratio * 100) + candidates.append( + JoinCandidate( + left_var=fs_i.var_name, + right_var=fs_j.var_name, + left_col=col_i.name, + right_col=col_j.name, + confidence=_CONFIDENCE_SAMPLE, + reason=( + f"matching dtype and overlapping sample values ({pct}% overlap)" + ), + ) + ) + return candidates + + +def _pass3_fuzzy( + fs_i: FileSchema, + fs_j: FileSchema, + existing_keys: set, +) -> List[JoinCandidate]: + """Pass 3: difflib fuzzy name ratio > 0.80. + + Only emits a candidate if no higher-confidence candidate already exists + for the same column pair. + + Args: + fs_i: Left file schema. + fs_j: Right file schema. + existing_keys: Set of candidate keys already found in Passes 1 & 2. + + Returns: + List of JoinCandidates with confidence=0.55. + """ + candidates: List[JoinCandidate] = [] + + for col_i in fs_i.columns: + for col_j in fs_j.columns: + key = _candidate_key(col_i.name, col_j.name, fs_i.var_name, fs_j.var_name) + if key in existing_keys: + continue # Already covered by a higher-confidence pass + + ratio = difflib.SequenceMatcher( + None, col_i.name.lower(), col_j.name.lower() + ).ratio() + + if ratio > _FUZZY_RATIO_THRESHOLD: + candidates.append( + JoinCandidate( + left_var=fs_i.var_name, + right_var=fs_j.var_name, + left_col=col_i.name, + right_col=col_j.name, + confidence=_CONFIDENCE_FUZZY, + reason=f"fuzzy name match (ratio: {ratio:.2f})", + ) + ) + return candidates + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +async def merge_schemas(file_metas: List[Dict[str, Any]]) -> MultiFileContext: + """Infers join candidates from stored workspace_files metadata. + + Runs three passes over every ordered file pair (i < j), deduplicates + by highest confidence, and returns a MultiFileContext with at most + ``_MAX_CANDIDATES`` join candidates. + + This function is intentionally pure (no I/O beyond CPU work) and + therefore needs no async I/O — the ``async def`` signature is kept for + API consistency with the async controller layer. + + Args: + file_metas: List of workspace_files row dicts, one per uploaded file. + Must include at minimum: file_name, file_path, file_type, + row_count, schema_json, upload_order. + + Returns: + MultiFileContext with populated files and join_candidates. + """ + if not file_metas: + logger.info("[SchemaMerger] No file metas provided — returning empty context.") + return MultiFileContext(files=[], join_candidates=[]) + + # Sort by upload_order ascending to assign df1, df2, ... deterministically + sorted_metas = sorted(file_metas, key=lambda m: m.get("upload_order", 1)) + + # Build FileSchema objects + file_schemas: List[FileSchema] = [] + for idx, meta in enumerate(sorted_metas, start=1): + var_name = f"df{idx}" + fs = _build_file_schema(meta, var_name) + file_schemas.append(fs) + logger.debug( + "[SchemaMerger] Built schema for %s (%s) with %d columns", + fs.file_name, + var_name, + len(fs.columns), + ) + + # Single file — no join candidates possible + if len(file_schemas) == 1: + logger.info( + "[SchemaMerger] Single file workspace — skipping join inference." + ) + return MultiFileContext(files=file_schemas, join_candidates=[]) + + # --- Run all three passes over every pair (i < j) --- + # best_candidates: key → (confidence, JoinCandidate) + best_candidates: Dict[str, Tuple[float, JoinCandidate]] = {} + + for i in range(len(file_schemas)): + for j in range(i + 1, len(file_schemas)): + fs_i = file_schemas[i] + fs_j = file_schemas[j] + + # Pass 1: Exact normalised match + for cand in _pass1_exact(fs_i, fs_j): + key = _candidate_key(cand.left_col, cand.right_col, cand.left_var, cand.right_var) + if key not in best_candidates or cand.confidence > best_candidates[key][0]: + best_candidates[key] = (cand.confidence, cand) + + # Pass 2: Dtype + sample overlap + for cand in _pass2_sample_overlap(fs_i, fs_j): + key = _candidate_key(cand.left_col, cand.right_col, cand.left_var, cand.right_var) + if key not in best_candidates or cand.confidence > best_candidates[key][0]: + best_candidates[key] = (cand.confidence, cand) + + # Pass 3: Fuzzy name match (skip already-found pairs) + existing_keys = set(best_candidates.keys()) + for cand in _pass3_fuzzy(fs_i, fs_j, existing_keys): + key = _candidate_key(cand.left_col, cand.right_col, cand.left_var, cand.right_var) + if key not in best_candidates or cand.confidence > best_candidates[key][0]: + best_candidates[key] = (cand.confidence, cand) + + # Deduplicate, sort by confidence descending, cap at _MAX_CANDIDATES + final_candidates = sorted( + (cand for _, cand in best_candidates.values()), + key=lambda c: c.confidence, + reverse=True, + )[:_MAX_CANDIDATES] + + logger.info( + "[SchemaMerger] %d file(s) → %d join candidate(s) (capped at %d).", + len(file_schemas), + len(final_candidates), + _MAX_CANDIDATES, + ) + + return MultiFileContext(files=file_schemas, join_candidates=final_candidates) diff --git a/backend/services/supabase_service.py b/backend/services/supabase_service.py index fb70c0f..4cfdcc0 100644 --- a/backend/services/supabase_service.py +++ b/backend/services/supabase_service.py @@ -185,8 +185,18 @@ def _sync() -> Dict[str, Any]: return await asyncio.get_running_loop().run_in_executor(None, _sync) -async def list_uploaded_files(workspace_id: Optional[str] = None) -> List[Dict[str, Any]]: - """Fetches all rows from the ``uploaded_files`` table. +async def list_uploaded_files( + user_id: Optional[str] = None, + workspace_id: Optional[str] = None, +) -> List[Dict[str, Any]]: + """Fetches file rows from the ``uploaded_files`` table scoped to a user. + + P2-04 fix: the ``user_id`` filter is now applied unconditionally when + provided, preventing cross-user file visibility in multi-tenant deployments. + + Args: + user_id: Authenticated user ID — rows are filtered to this user only. + workspace_id: Optional workspace scope for additional filtering. Returns: List[Dict[str, Any]]: List of dataset metadata rows. @@ -194,6 +204,8 @@ async def list_uploaded_files(workspace_id: Optional[str] = None) -> List[Dict[s def _sync() -> List[Dict[str, Any]]: client = _make_service_client() query = client.table("uploaded_files").select("*").order("created_at", desc=True) + if user_id: + query = query.eq("user_id", user_id) if workspace_id: query = query.eq("workspace_id", workspace_id) response = query.execute() @@ -288,7 +300,7 @@ def _sync() -> Dict[str, Any]: "insights": insights, "execution_logs": execution_logs, "status": status, - "completed_at": datetime.datetime.utcnow().isoformat(), + "completed_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), } try: response = ( @@ -358,7 +370,7 @@ def _sync() -> List[Dict[str, Any]]: try: query = ( client.table("agent_runs") - .select("id, query, file_names, rounds, status, created_at, completed_at, eval_metrics, user_id, workspace_id") + .select("id, query, file_names, rounds, status, created_at, completed_at, user_id, workspace_id") .order("created_at", desc=True) .limit(limit) ) @@ -383,56 +395,6 @@ def _sync() -> List[Dict[str, Any]]: return await asyncio.get_running_loop().run_in_executor(None, _sync) -async def update_agent_run_metrics( - run_id: str, - metrics: Dict[str, Any], - total_run_ms: int, - complexity: str, -) -> Dict[str, Any]: - """Persists evaluation metrics onto an existing agent_run row. - - Args: - run_id (str): Unique run identifier matching an existing agent_runs row. - metrics (Dict[str, Any]): RunMetrics.summary() output. - total_run_ms (int): Total wall-clock duration of the run in milliseconds. - complexity (str): Task complexity tag — ``"easy"`` or ``"hard"``. - - Returns: - Dict[str, Any]: The updated row, or empty dict on failure. - """ - def _sync() -> Dict[str, Any]: - client = _make_service_client() - updates = { - "eval_metrics": { - **metrics, - "total_run_ms": total_run_ms, - "complexity": complexity, - } - } - try: - response = ( - client.table("agent_runs") - .update(updates) - .eq("id", run_id) - .execute() - ) - logger.info( - "Persisted eval_metrics for run_id=%s — complexity=%s, total_ms=%d", - run_id, - complexity, - total_run_ms, - ) - return response.data[0] if response.data else {} - except Exception as exc: # pylint: disable=broad-except - logger.warning( - "Could not persist eval_metrics for run_id=%s (column may not exist yet): %s", - run_id, - exc, - ) - return {} - - return await asyncio.get_running_loop().run_in_executor(None, _sync) - # --------------------------------------------------------------------------- # Deep Research (DS-STAR+) tracking @@ -547,7 +509,7 @@ def _sync() -> None: client.table("sub_questions").update({ "status": status, "result_run_id": result_run_id, - "completed_at": datetime.datetime.utcnow().isoformat() + "completed_at": datetime.datetime.now(datetime.timezone.utc).isoformat() }).eq("id", sq_id).execute() except Exception as exc: # pylint: disable=broad-except logger.warning("[Supabase] Could not link sub-question %s: %s", sq_id, exc) @@ -587,7 +549,7 @@ def _sync() -> None: "key_findings": key_findings or [], "caveats": caveats or [], "total_ms": total_ms, - "completed_at": datetime.datetime.utcnow().isoformat() + "completed_at": datetime.datetime.now(datetime.timezone.utc).isoformat() } try: client.table("reports").update(updates).eq("id", report_id).execute() @@ -702,3 +664,168 @@ def _sync() -> Dict[str, Any]: return {} return await asyncio.get_running_loop().run_in_executor(None, _sync) + + +# --------------------------------------------------------------------------- +# workspace_files operations (multi-file join feature) +# --------------------------------------------------------------------------- + +async def insert_workspace_file(record: Dict[str, Any]) -> Dict[str, Any]: + """Inserts one row into the ``workspace_files`` table. + + Args: + record: Dict matching workspace_files schema keys: + workspace_id, user_id, file_name, file_path, file_type, + row_count (optional), schema_json (optional), upload_order. + + Returns: + The inserted row as returned by Supabase. + + Raises: + RuntimeError: If the insert fails. + """ + def _sync() -> Dict[str, Any]: + client = _make_service_client() + try: + response = ( + client.table("workspace_files") + .insert(record) + .execute() + ) + if not response.data: + raise RuntimeError( + f"workspace_files insert returned no data for record: {record}" + ) + logger.info( + "[Supabase] Inserted workspace_file id=%s file=%s order=%s", + response.data[0].get("id"), + record.get("file_name"), + record.get("upload_order"), + ) + return response.data[0] + except Exception as exc: # pylint: disable=broad-except + logger.warning("[Supabase] Could not insert workspace_file: %s", exc) + raise RuntimeError(str(exc)) from exc + + return await asyncio.get_running_loop().run_in_executor(None, _sync) + + +async def list_workspace_files( + workspace_id: str, + user_id: str, +) -> List[Dict[str, Any]]: + """Fetches all workspace_files rows for a given workspace, ordered by upload_order ASC. + + Args: + workspace_id: The workspace UUID to filter by. + user_id: The authenticated user UUID — enforces row-level ownership. + + Returns: + List of workspace_files rows ordered by upload_order ascending. + """ + def _sync() -> List[Dict[str, Any]]: + client = _make_service_client() + try: + response = ( + client.table("workspace_files") + .select("*") + .eq("workspace_id", workspace_id) + .eq("user_id", user_id) + .order("upload_order", desc=False) + .execute() + ) + return response.data or [] + except Exception as exc: # pylint: disable=broad-except + logger.warning( + "[Supabase] Could not list workspace_files for workspace=%s: %s", + workspace_id, + exc, + ) + return [] + + return await asyncio.get_running_loop().run_in_executor(None, _sync) + + +async def count_workspace_files(workspace_id: str, user_id: str) -> int: + """Returns the count of existing workspace_files rows for a workspace. + + Used to determine the next upload_order value (count + 1). + + Args: + workspace_id: The workspace UUID. + user_id: The authenticated user UUID. + + Returns: + Integer count of existing rows (0 if none). + """ + def _sync() -> int: + client = _make_service_client() + try: + response = ( + client.table("workspace_files") + .select("id", count="exact") + .eq("workspace_id", workspace_id) + .eq("user_id", user_id) + .execute() + ) + return response.count or 0 + except Exception as exc: # pylint: disable=broad-except + logger.warning( + "[Supabase] Could not count workspace_files for workspace=%s: %s", + workspace_id, + exc, + ) + return 0 + + return await asyncio.get_running_loop().run_in_executor(None, _sync) + + +async def delete_workspace_file( + file_id: str, + workspace_id: str, + user_id: str, +) -> bool: + """Deletes a single workspace_files row after verifying ownership. + + Args: + file_id: UUID of the workspace_files row to delete. + workspace_id: Must match the row's workspace_id (ownership check). + user_id: Must match the row's user_id (ownership check). + + Returns: + True if a row was deleted, False if not found or not owned by user. + """ + def _sync() -> bool: + client = _make_service_client() + try: + response = ( + client.table("workspace_files") + .delete() + .eq("id", file_id) + .eq("workspace_id", workspace_id) + .eq("user_id", user_id) + .execute() + ) + deleted = bool(response.data) + if deleted: + logger.info( + "[Supabase] Deleted workspace_file id=%s from workspace=%s", + file_id, + workspace_id, + ) + else: + logger.warning( + "[Supabase] workspace_file id=%s not found or not owned by user=%s", + file_id, + user_id, + ) + return deleted + except Exception as exc: # pylint: disable=broad-except + logger.warning( + "[Supabase] Could not delete workspace_file id=%s: %s", + file_id, + exc, + ) + return False + + return await asyncio.get_running_loop().run_in_executor(None, _sync) diff --git a/backend/services/upload_service.py b/backend/services/upload_service.py index 3b13f22..80499eb 100644 --- a/backend/services/upload_service.py +++ b/backend/services/upload_service.py @@ -10,6 +10,7 @@ """ import logging +import threading from typing import Dict, Optional, Tuple from fastapi import UploadFile @@ -20,9 +21,78 @@ _ANON_SESSION = "__anon__" +# --------------------------------------------------------------------------- +# Magic-byte MIME validation (P1-2 security fix) +# --------------------------------------------------------------------------- + +try: + import magic as _magic # python-magic-bin (already in requirements.txt) + _MAGIC_AVAILABLE = True +except ImportError: + _magic = None # type: ignore[assignment] + _MAGIC_AVAILABLE = False + logger.warning( + "[UploadService] python-magic not available — MIME validation disabled. " + "Install python-magic-bin to enable magic-byte file type checking." + ) + +# Allowlist: extension → set of acceptable MIME types returned by libmagic. +# Only file types the platform actually needs to process are permitted. +_ALLOWED_MIME_BY_EXT: Dict[str, set] = { + "csv": {"text/plain", "text/csv", "application/csv"}, + "tsv": {"text/plain", "text/tab-separated-values"}, + "txt": {"text/plain"}, + "json": {"application/json", "text/plain"}, + "xlsx": { + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/zip", # libmagic sees XLSX as zip + }, + "xls": {"application/vnd.ms-excel"}, + "parquet": {"application/octet-stream"}, + "pdf": {"application/pdf"}, +} + + +def _validate_mime(filename: str, content: bytes) -> None: + """Raises ``ValueError`` when the file's magic bytes contradict its extension. + + Protects against extension-spoofing attacks where a malicious payload is + disguised with a benign extension (e.g. an ELF binary named ``data.csv``). + Only validates file types present in ``_ALLOWED_MIME_BY_EXT``; unknown + extensions are passed through without magic-byte verification (and therefore + cannot be spoofed into an accepted type). + + Args: + filename: Original upload filename (used to derive the extension). + content: Raw file bytes (only the first 2 KB are inspected). + + Raises: + ValueError: If the MIME type detected by libmagic is not in the + allowlist for the declared extension. + """ + if not _MAGIC_AVAILABLE or not filename: + return + + ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + allowed_mimes = _ALLOWED_MIME_BY_EXT.get(ext) + if allowed_mimes is None: + # Unknown extension — reject outright (not in the allowlist at all) + raise ValueError( + f"Unsupported file extension '.{ext}'. " + f"Accepted types: {sorted(_ALLOWED_MIME_BY_EXT.keys())}" + ) + + detected = _magic.from_buffer(content[:2048], mime=True) + if detected not in allowed_mimes: + raise ValueError( + f"File '{filename}' rejected: extension claims '.{ext}' but magic " + f"bytes indicate '{detected}'. This may be a spoofed upload." + ) + # Two-level in-memory cache: {session_id → {filename → bytes}} # Never access this dict directly from outside this module. _FILE_CACHE: Dict[str, Dict[str, bytes]] = {} +_CACHE_LOCK = threading.Lock() async def save_upload_file( @@ -58,7 +128,16 @@ async def save_upload_file( # Store in the session bucket — never touching other sessions content = b"".join(content_chunks) - _FILE_CACHE.setdefault(session_id, {})[file.filename] = content + + # P1-2 MIME validation: verify magic bytes match the declared extension + # before accepting the file into the session cache or Supabase. + try: + _validate_mime(file.filename, content) + except ValueError as mime_err: + raise ValueError(str(mime_err)) from mime_err + + with _CACHE_LOCK: + _FILE_CACHE.setdefault(session_id, {})[file.filename] = content file_format = ( file.filename.rsplit(".", 1)[-1].lower() @@ -85,7 +164,7 @@ async def save_upload_file( } await insert_file_record(record) except Exception as e: - logger.warning("Failed to persist file %s to Supabase: %s", file.filename, e) + logger.debug("Failed to persist file %s to Supabase Storage (non-critical): %s", file.filename, e) logger.info( "Uploaded File Metadata - Name: %s, Format: %s, Session: %s", @@ -110,7 +189,8 @@ def get_file_content( Returns: Raw bytes if found, else None. """ - return _FILE_CACHE.get(session_id, {}).get(filename) + with _CACHE_LOCK: + return _FILE_CACHE.get(session_id, {}).get(filename) def get_session_files(session_id: str = _ANON_SESSION) -> Dict[str, bytes]: @@ -125,7 +205,8 @@ def get_session_files(session_id: str = _ANON_SESSION) -> Dict[str, bytes]: Returns: Dict mapping filename → bytes for the session. """ - return dict(_FILE_CACHE.get(session_id, {})) + with _CACHE_LOCK: + return dict(_FILE_CACHE.get(session_id, {})) def clear_file_cache(session_id: str = _ANON_SESSION) -> None: @@ -135,5 +216,6 @@ def clear_file_cache(session_id: str = _ANON_SESSION) -> None: session_id: Session to wipe. Only that session's data is removed; other sessions are unaffected. """ - _FILE_CACHE.pop(session_id, None) + with _CACHE_LOCK: + _FILE_CACHE.pop(session_id, None) logger.info("[UploadService] Cleared file cache for session: %s", session_id) diff --git a/backend/tests/test_schema_merger.py b/backend/tests/test_schema_merger.py new file mode 100644 index 0000000..599d4c0 --- /dev/null +++ b/backend/tests/test_schema_merger.py @@ -0,0 +1,296 @@ +"""Tests for schema_merger — the join key inference service. + +Covers: + - Exact name match across two files + - Fuzzy name match fallback + - Dtype+sample overlap match + - No match (empty candidates) + - Three-file scenario (all pairs checked) + - Single file returns empty candidates + - Case-insensitive exact match +""" + +import asyncio +import pytest + +from models.multi_file_context import ColumnMeta, FileSchema, MultiFileContext + + +# --------------------------------------------------------------------------- +# Helpers to build minimal schema metadata rows +# --------------------------------------------------------------------------- + +def _make_row( + file_name: str, + columns: list[dict], + var_name: str | None = None, + file_path: str = "/workspace/ws1/file.csv", + file_type: str = "csv", + row_count: int = 100, + order: int = 1, +) -> dict: + """Produces a workspace_files row dict suitable for merge_schemas().""" + schema_json = { + "columns": [c["name"] for c in columns], + "dtypes": {c["name"]: c.get("dtype", "object") for c in columns}, + "sample_rows": [ + {c["name"]: (c.get("samples", ["x"])[0] if c.get("samples") else None) for c in columns} + ], + "row_count": row_count, + } + return { + "file_name": file_name, + "file_path": file_path, + "file_type": file_type, + "row_count": row_count, + "schema_json": schema_json, + "upload_order": order, + } + + +# --------------------------------------------------------------------------- +# Async wrapper +# --------------------------------------------------------------------------- + +def run(coro): + return asyncio.get_event_loop().run_until_complete(coro) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +class TestSchemaMergerExactMatch: + """Exact column name match should produce a high-confidence candidate.""" + + def test_exact_name_match_produces_candidate(self): + from services.schema_merger import merge_schemas + + rows = [ + _make_row("orders.csv", [{"name": "customer_id", "dtype": "int64"}], order=1), + _make_row("customers.csv", [{"name": "customer_id", "dtype": "int64"}], order=2), + ] + ctx = run(merge_schemas(rows)) + + assert isinstance(ctx, MultiFileContext) + assert len(ctx.files) == 2 + assert len(ctx.join_candidates) >= 1 + + top = ctx.join_candidates[0] + assert top.left_col == "customer_id" + assert top.right_col == "customer_id" + assert top.confidence >= 0.8, f"Expected >= 0.8, got {top.confidence}" + + def test_exact_match_case_insensitive(self): + """Column names that differ only in case should still match.""" + from services.schema_merger import merge_schemas + + rows = [ + _make_row("a.csv", [{"name": "OrderID", "dtype": "int64"}], order=1), + _make_row("b.csv", [{"name": "orderid", "dtype": "int64"}], order=2), + ] + ctx = run(merge_schemas(rows)) + assert any( + c.confidence >= 0.7 + for c in ctx.join_candidates + ), "Expected at least one candidate with conf >= 0.7 for case-insensitive match" + + +class TestSchemaMergerNoMatch: + """Completely disjoint schemas should yield empty or low-confidence candidates.""" + + def test_no_shared_columns_yields_empty_or_low(self): + from services.schema_merger import merge_schemas + + rows = [ + _make_row("a.csv", [{"name": "foo", "dtype": "object"}], order=1), + _make_row("b.csv", [{"name": "bar", "dtype": "object"}], order=2), + ] + ctx = run(merge_schemas(rows)) + + high_conf = [c for c in ctx.join_candidates if c.confidence >= 0.5] + assert len(high_conf) == 0, ( + f"Expected no high-confidence candidates for disjoint schemas, got {high_conf}" + ) + + +class TestSchemaMergerDtypeOverlap: + """When column names differ but dtypes and sample values overlap, a candidate should appear.""" + + def test_sample_value_overlap_detected(self): + from services.schema_merger import merge_schemas + + shared_ids = ["101", "102", "103", "104", "105"] + rows = [ + _make_row( + "orders.csv", + [{"name": "order_cust", "dtype": "int64", "samples": shared_ids}], + order=1, + ), + _make_row( + "customers.csv", + [{"name": "client_num", "dtype": "int64", "samples": shared_ids}], + order=2, + ), + ] + ctx = run(merge_schemas(rows)) + + # There should be at least one candidate — may not be high-confidence + assert len(ctx.join_candidates) >= 1, "Expected at least one candidate via sample overlap" + + +class TestSchemaMergerThreeFiles: + """Three files should produce candidates across all pairs.""" + + def test_three_file_all_pairs_checked(self): + from services.schema_merger import merge_schemas + + rows = [ + _make_row("a.csv", [{"name": "id", "dtype": "int64"}], order=1), + _make_row("b.csv", [{"name": "id", "dtype": "int64"}], order=2), + _make_row("c.csv", [{"name": "id", "dtype": "int64"}], order=3), + ] + ctx = run(merge_schemas(rows)) + + assert len(ctx.files) == 3 + # Pairs: (df1,df2), (df1,df3), (df2,df3) — each 'id'<->'id' should match + assert len(ctx.join_candidates) >= 3, ( + f"Expected >= 3 candidates for 3-file exact-match scenario, got {ctx.join_candidates}" + ) + + +class TestSchemaMergerSingleFile: + """Single-file workspace should return 0 candidates.""" + + def test_single_file_returns_empty_candidates(self): + from services.schema_merger import merge_schemas + + rows = [_make_row("only.csv", [{"name": "col_a", "dtype": "object"}], order=1)] + ctx = run(merge_schemas(rows)) + + assert len(ctx.files) == 1 + assert len(ctx.join_candidates) == 0 + + +class TestSchemaMergerOrdering: + """Candidates should be sorted by confidence descending.""" + + def test_candidates_sorted_by_confidence(self): + from services.schema_merger import merge_schemas + + rows = [ + _make_row("x.csv", [ + {"name": "key", "dtype": "int64"}, + {"name": "fuzzkey", "dtype": "int64"}, + ], order=1), + _make_row("y.csv", [ + {"name": "key", "dtype": "int64"}, + {"name": "fuzzy_key", "dtype": "int64"}, + ], order=2), + ] + ctx = run(merge_schemas(rows)) + + confidences = [c.confidence for c in ctx.join_candidates] + assert confidences == sorted(confidences, reverse=True), ( + f"Candidates not sorted by confidence: {confidences}" + ) + + +class TestSchemaMergerVariableNames: + """Variable names assigned must be df1, df2, ... in upload order.""" + + def test_var_names_assigned_sequentially(self): + from services.schema_merger import merge_schemas + + rows = [ + _make_row("first.csv", [{"name": "x"}], order=1), + _make_row("second.csv", [{"name": "x"}], order=2), + _make_row("third.csv", [{"name": "x"}], order=3), + ] + ctx = run(merge_schemas(rows)) + + var_names = [f.var_name for f in ctx.files] + assert var_names == ["df1", "df2", "df3"], f"Unexpected var_names: {var_names}" + + +class TestMultiFileContextToPromptStr: + """to_prompt_str() must produce a non-empty, well-formed prompt block.""" + + def test_prompt_str_contains_all_files(self): + from services.schema_merger import merge_schemas + + rows = [ + _make_row("alpha.csv", [{"name": "id", "dtype": "int64"}], order=1), + _make_row("beta.csv", [{"name": "id", "dtype": "int64"}], order=2), + ] + ctx = run(merge_schemas(rows)) + prompt = ctx.to_prompt_str() + + assert "alpha.csv" in prompt + assert "beta.csv" in prompt + assert "JOIN CANDIDATES" in prompt + + def test_prompt_str_under_token_budget(self): + """Rough check: prompt should not exceed 3000 chars for typical inputs.""" + from services.schema_merger import merge_schemas + + cols = [{"name": f"col_{i}", "dtype": "object"} for i in range(40)] + rows = [ + _make_row("wide1.csv", cols, order=1), + _make_row("wide2.csv", cols, order=2), + ] + ctx = run(merge_schemas(rows)) + prompt = ctx.to_prompt_str() + + assert len(prompt) <= 3000, ( + f"to_prompt_str() too long ({len(prompt)} chars) — likely not truncating columns" + ) + + +class TestMultiFileContextToReaderHeader: + """to_reader_header() must produce valid-looking Python for each file type.""" + + def _make_ctx(self, files_specs): + files = [] + for i, (fname, ftype, fpath) in enumerate(files_specs, start=1): + fs = FileSchema( + var_name=f"df{i}", + file_name=fname, + file_path=fpath, + file_type=ftype, + row_count=100, + columns=[ColumnMeta(name="col", dtype="int64")], + ) + files.append(fs) + return MultiFileContext(files=files) + + def test_csv_reader(self): + ctx = self._make_ctx([("data.csv", "csv", "/ws/data.csv")]) + header = ctx.to_reader_header(workspace_id="ws1") + assert "pd.read_csv" in header + assert "df1" in header + + def test_excel_reader(self): + ctx = self._make_ctx([("data.xlsx", "xlsx", "/ws/data.xlsx")]) + header = ctx.to_reader_header(workspace_id="ws1") + assert "pd.read_excel" in header + + def test_parquet_reader(self): + ctx = self._make_ctx([("data.parquet", "parquet", "/ws/data.parquet")]) + header = ctx.to_reader_header(workspace_id="ws1") + assert "pd.read_parquet" in header + + def test_markdown_fallback(self): + ctx = self._make_ctx([("notes.md", "md", "/ws/notes.md")]) + header = ctx.to_reader_header(workspace_id="ws1") + assert "startswith('|')" in header + assert "df1" in header + + def test_multi_file_header_contains_verification_prints(self): + ctx = self._make_ctx([ + ("a.csv", "csv", "/ws/a.csv"), + ("b.xlsx", "xlsx", "/ws/b.xlsx"), + ]) + header = ctx.to_reader_header(workspace_id="ws1") + assert "df1 shape" in header + assert "df2 shape" in header diff --git a/backend/tests/test_upload_controller.py b/backend/tests/test_upload_controller.py new file mode 100644 index 0000000..0609e21 --- /dev/null +++ b/backend/tests/test_upload_controller.py @@ -0,0 +1,314 @@ +"""Tests for the workspace upload and delete controller functions. + +Covers: + - handle_workspace_upload: accepted + rejected files + - handle_workspace_upload: parser failure is non-fatal + - handle_workspace_upload: empty file list returns empty MultiFileContext + - handle_delete_workspace_file: success path returns updated context + - handle_delete_workspace_file: raises ValueError for unknown file_id + - handle_upload (legacy): backward-compat path unchanged +""" + +import io +import os +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + +from fastapi import UploadFile +from httpx import Headers + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_upload_file(filename: str, content: bytes = b"col_a,col_b\n1,2\n") -> UploadFile: + """Creates a minimal UploadFile mock that behaves like a real upload.""" + file_like = io.BytesIO(content) + # FastAPI's UploadFile expects a SpooledTemporaryFile; BytesIO is sufficient for tests. + uf = MagicMock(spec=UploadFile) + uf.filename = filename + uf.read = AsyncMock(return_value=content) + uf.seek = AsyncMock() + return uf + + +# --------------------------------------------------------------------------- +# validate_file_metadata stub — always returns None (no issue) +# --------------------------------------------------------------------------- + +_VALIDATE_PATCH = "api.controllers.upload_controller.validate_file_metadata" +_SAVE_PATCH = "api.controllers.upload_controller.save_upload_file" +_INSERT_PATCH = "services.supabase_service.insert_workspace_file" +_COUNT_PATCH = "services.supabase_service.count_workspace_files" +_LIST_PATCH = "services.supabase_service.list_workspace_files" +_MERGE_PATCH = "api.controllers.upload_controller.merge_schemas" +_PARSER_PATCH = "api.controllers.upload_controller._get_parser_for" + + +# --------------------------------------------------------------------------- +# handle_workspace_upload tests +# --------------------------------------------------------------------------- + +class TestHandleWorkspaceUpload: + """Tests for the multi-file workspace upload controller.""" + + @pytest.mark.asyncio + async def test_single_file_accepted(self, tmp_path): + """A valid single CSV upload should appear in accepted_files.""" + from api.controllers.upload_controller import handle_workspace_upload + + file = _make_upload_file("sales.csv") + + mock_mfc = MagicMock() + mock_mfc.join_candidates = [] + mock_mfc.model_dump.return_value = {"files": [], "join_candidates": []} + + with ( + patch(_VALIDATE_PATCH, new=AsyncMock(return_value=None)), + patch(_SAVE_PATCH, new=AsyncMock()), + patch(_COUNT_PATCH, new=AsyncMock(return_value=0)), + patch(_INSERT_PATCH, new=AsyncMock()), + patch(_LIST_PATCH, new=AsyncMock(return_value=[{"id": "abc", "file_name": "sales.csv"}])), + patch(_MERGE_PATCH, new=AsyncMock(return_value=mock_mfc)), + patch(_PARSER_PATCH, return_value=None), + patch("os.makedirs"), + patch("builtins.open", MagicMock()), + ): + result = await handle_workspace_upload( + files=[file], + workspace_id="ws-001", + user_id="user-001", + session_id="sess-001", + ) + + assert len(result["accepted_files"]) == 1 + assert result["accepted_files"][0].filename == "sales.csv" + assert result["rejected_files"] == [] + + @pytest.mark.asyncio + async def test_rejected_on_validation_failure(self): + """A file that fails MIME validation should land in rejected_files.""" + from api.controllers.upload_controller import handle_workspace_upload + + file = _make_upload_file("malicious.exe") + + mock_mfc = MagicMock() + mock_mfc.join_candidates = [] + mock_mfc.model_dump.return_value = {"files": [], "join_candidates": []} + + with ( + patch(_VALIDATE_PATCH, new=AsyncMock(return_value="Unsupported file type: exe")), + patch(_LIST_PATCH, new=AsyncMock(return_value=[])), + patch(_MERGE_PATCH, new=AsyncMock(return_value=mock_mfc)), + ): + result = await handle_workspace_upload( + files=[file], + workspace_id="ws-001", + user_id="user-001", + ) + + assert len(result["rejected_files"]) == 1 + assert "exe" in result["rejected_files"][0].reason.lower() + assert result["accepted_files"] == [] + + @pytest.mark.asyncio + async def test_parser_failure_is_nonfatal(self): + """If the parser raises an exception, the file is still accepted (schema_json empty).""" + from api.controllers.upload_controller import handle_workspace_upload + + file = _make_upload_file("data.csv") + + def _exploding_parser(filename, content): + raise RuntimeError("Parser exploded") + + mock_mfc = MagicMock() + mock_mfc.join_candidates = [] + mock_mfc.model_dump.return_value = {"files": [], "join_candidates": []} + + with ( + patch(_VALIDATE_PATCH, new=AsyncMock(return_value=None)), + patch(_SAVE_PATCH, new=AsyncMock()), + patch(_COUNT_PATCH, new=AsyncMock(return_value=0)), + patch(_INSERT_PATCH, new=AsyncMock()), + patch(_LIST_PATCH, new=AsyncMock(return_value=[])), + patch(_MERGE_PATCH, new=AsyncMock(return_value=mock_mfc)), + patch(_PARSER_PATCH, return_value=_exploding_parser), + patch("os.makedirs"), + patch("builtins.open", MagicMock()), + ): + result = await handle_workspace_upload( + files=[file], + workspace_id="ws-001", + user_id="user-001", + ) + + # File should still be accepted despite parser failure + assert len(result["accepted_files"]) == 1 + + @pytest.mark.asyncio + async def test_empty_file_list_returns_empty(self): + """Uploading no files should return empty lists and an empty MultiFileContext.""" + from api.controllers.upload_controller import handle_workspace_upload + + mock_mfc = MagicMock() + mock_mfc.join_candidates = [] + mock_mfc.model_dump.return_value = {"files": [], "join_candidates": []} + + with ( + patch(_LIST_PATCH, new=AsyncMock(return_value=[])), + patch(_MERGE_PATCH, new=AsyncMock(return_value=mock_mfc)), + ): + result = await handle_workspace_upload( + files=[], + workspace_id="ws-001", + user_id="user-001", + ) + + assert result["accepted_files"] == [] + assert result["rejected_files"] == [] + + @pytest.mark.asyncio + async def test_multi_file_upload_order_increments(self): + """upload_order should be 1 for first file and 2 for second in the same call.""" + from api.controllers.upload_controller import handle_workspace_upload + + inserted_orders = [] + + async def _fake_insert(record): + inserted_orders.append(record["upload_order"]) + + count_call = 0 + async def _fake_count(**kwargs): + nonlocal count_call + count_call += 1 + # Simulate: 0 existing before first file, 1 before second + return count_call - 1 + + mock_mfc = MagicMock() + mock_mfc.join_candidates = [] + mock_mfc.model_dump.return_value = {"files": [], "join_candidates": []} + + file1 = _make_upload_file("a.csv") + file2 = _make_upload_file("b.csv") + + with ( + patch(_VALIDATE_PATCH, new=AsyncMock(return_value=None)), + patch(_SAVE_PATCH, new=AsyncMock()), + patch(_COUNT_PATCH, new=AsyncMock(side_effect=_fake_count)), + patch("services.supabase_service.insert_workspace_file", new=AsyncMock(side_effect=_fake_insert)), + patch(_LIST_PATCH, new=AsyncMock(return_value=[])), + patch(_MERGE_PATCH, new=AsyncMock(return_value=mock_mfc)), + patch(_PARSER_PATCH, return_value=None), + patch("os.makedirs"), + patch("builtins.open", MagicMock()), + ): + await handle_workspace_upload( + files=[file1, file2], + workspace_id="ws-001", + user_id="user-001", + ) + + assert inserted_orders == [1, 2], f"Expected [1, 2], got {inserted_orders}" + + +# --------------------------------------------------------------------------- +# handle_delete_workspace_file tests +# --------------------------------------------------------------------------- + +class TestHandleDeleteWorkspaceFile: + """Tests for the file deletion controller.""" + + @pytest.mark.asyncio + async def test_delete_success_returns_updated_context(self): + """Deleting an existing file returns deleted=True and the refreshed context.""" + from api.controllers.upload_controller import handle_delete_workspace_file + + mock_mfc = MagicMock() + mock_mfc.join_candidates = [] + mock_mfc.model_dump.return_value = {"files": [], "join_candidates": []} + + _LIST_SIDE_EFFECTS = [ + # First call (pre-delete): one file exists + [{"id": "file-123", "file_name": "orders.csv", "file_path": "/tmp/orders.csv"}], + # Second call (post-delete): workspace is empty + [], + ] + + with ( + patch("api.controllers.upload_controller.list_workspace_files", + new=AsyncMock(side_effect=_LIST_SIDE_EFFECTS)), + patch("api.controllers.upload_controller.delete_workspace_file", + new=AsyncMock(return_value=True)), + patch("api.controllers.upload_controller.merge_schemas", + new=AsyncMock(return_value=mock_mfc)), + patch("os.path.isfile", return_value=False), # skip physical delete + ): + result = await handle_delete_workspace_file( + file_id="file-123", + workspace_id="ws-001", + user_id="user-001", + ) + + assert result["deleted"] is True + assert "multi_file_context" in result + + @pytest.mark.asyncio + async def test_delete_unknown_file_raises_value_error(self): + """Attempting to delete a file_id not in the workspace raises ValueError.""" + from api.controllers.upload_controller import handle_delete_workspace_file + + with ( + patch("api.controllers.upload_controller.list_workspace_files", + new=AsyncMock(return_value=[])), # empty workspace + ): + with pytest.raises(ValueError, match="not found"): + await handle_delete_workspace_file( + file_id="nonexistent-id", + workspace_id="ws-001", + user_id="user-001", + ) + + +# --------------------------------------------------------------------------- +# handle_upload (legacy backward-compat) tests +# --------------------------------------------------------------------------- + +class TestHandleUploadLegacy: + """Ensures the legacy /upload path is still functional.""" + + @pytest.mark.asyncio + async def test_legacy_upload_returns_accepted(self): + from api.controllers.upload_controller import handle_upload + + file = _make_upload_file("legacy.csv") + + with ( + patch(_VALIDATE_PATCH, new=AsyncMock(return_value=None)), + patch(_SAVE_PATCH, new=AsyncMock()), + ): + response = await handle_upload( + files=[file], + session_id="sess-legacy", + user_id="user-001", + ) + + assert len(response.accepted_files) == 1 + assert response.accepted_files[0].filename == "legacy.csv" + assert response.rejected_files == [] + + @pytest.mark.asyncio + async def test_legacy_upload_rejects_on_validation_error(self): + from api.controllers.upload_controller import handle_upload + + file = _make_upload_file("bad.exe") + + with patch(_VALIDATE_PATCH, new=AsyncMock(return_value="Unsupported type")): + response = await handle_upload( + files=[file], + session_id="sess-legacy", + user_id="user-001", + ) + + assert len(response.rejected_files) == 1 + assert response.accepted_files == [] diff --git a/backend/utils/helpers.py b/backend/utils/helpers.py index a6aabb9..7fe9ab8 100644 --- a/backend/utils/helpers.py +++ b/backend/utils/helpers.py @@ -1,19 +1,9 @@ """General utility functions and helpers.""" import math -import uuid from typing import Any -def generate_id() -> str: - """Generates a random unique identifier string. - - Returns: - str: A randomly generated UUID string. - """ - return str(uuid.uuid4()) - - def sanitize_floats(obj: Any) -> Any: """Recursively replaces non-JSON-compliant float values with None. diff --git a/eslint.config.js b/eslint.config.js index 4fa125d..5aa17d6 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -23,7 +23,17 @@ export default defineConfig([ }, }, rules: { - 'no-unused-vars': ['error', { varsIgnorePattern: '^[A-Z_]' }], + 'no-unused-vars': ['error', { + varsIgnorePattern: '^[A-Z_]', + argsIgnorePattern: '^_|^Icon$', + caughtErrorsIgnorePattern: '^_', + }], + }, + }, + { + files: ['tailwind.config.js', 'postcss.config.js'], + languageOptions: { + globals: globals.node, }, }, ]) diff --git a/scratch/backfill_user_id.py b/scratch/backfill_user_id.py deleted file mode 100644 index 985323c..0000000 --- a/scratch/backfill_user_id.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -from supabase import create_client -from dotenv import load_dotenv - -load_dotenv('e:/Agentloop/backend/.env') - -url = os.environ.get("SUPABASE_URL") -key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY") - -client = create_client(url, key) - -print("Backfilling user_id for agent_runs based on workspace_id...") -try: - # Get all workspaces - ws_res = client.table("workspaces").select("id, user_id").execute() - workspaces = ws_res.data - ws_map = {ws['id']: ws['user_id'] for ws in workspaces} - - # Get all agent_runs with None user_id and not None workspace_id - runs_res = client.table("agent_runs").select("id, workspace_id").is_("user_id", "null").not_.is_("workspace_id", "null").execute() - runs = runs_res.data - - count = 0 - for r in runs: - ws_id = r.get('workspace_id') - if ws_id in ws_map: - user_id = ws_map[ws_id] - client.table("agent_runs").update({"user_id": user_id}).eq("id", r['id']).execute() - count += 1 - - print(f"Updated {count} agent_runs.") - - # Also backfill uploaded_files (if possible, though I saw it wasn't in schema cache earlier) - try: - files_res = client.table("uploaded_files").select("id, workspace_id").is_("user_id", "null").not_.is_("workspace_id", "null").execute() - files = files_res.data - fc = 0 - for f in files: - ws_id = f.get('workspace_id') - if ws_id in ws_map: - user_id = ws_map[ws_id] - client.table("uploaded_files").update({"user_id": user_id}).eq("id", f['id']).execute() - fc += 1 - print(f"Updated {fc} uploaded_files.") - except Exception as e: - print("uploaded_files backfill error:", e) - -except Exception as e: - print("Error:", e) diff --git a/scratch/test_db_data.py b/scratch/test_db_data.py deleted file mode 100644 index 94c0113..0000000 --- a/scratch/test_db_data.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import sys -from supabase import create_client -from dotenv import load_dotenv - -load_dotenv('e:/Agentloop/backend/.env') - -url = os.environ.get("SUPABASE_URL") -key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY") - -client = create_client(url, key) - -print("Fetching agent_runs...") -try: - res = client.table("agent_runs").select("id, user_id, workspace_id, created_at, query").execute() - runs = res.data - print(f"Total runs: {len(runs)}") - for r in runs: - print(r) -except Exception as e: - print("Error:", e) diff --git a/scratch/test_dump.py b/scratch/test_dump.py deleted file mode 100644 index 321d281..0000000 --- a/scratch/test_dump.py +++ /dev/null @@ -1,38 +0,0 @@ -import asyncio -import sys -import os -import traceback - -sys.path.insert(0, os.path.abspath('backend')) - -from core.planner.planner_agent import PlannerAgent -import logging - -logging.basicConfig(level=logging.DEBUG) - -async def dump_tasks(): - await asyncio.sleep(5) - print("\n--- DUMPING TASKS ---") - for task in asyncio.all_tasks(): - print(f"Task: {task.get_name()}") - task.print_stack() - print("---------------------\n") - sys.exit(1) - -async def main(): - asyncio.create_task(dump_tasks()) - - agent = PlannerAgent() - print("Agent created") - - # We will step into get_chain - print("Calling planner create_plan") - - try: - plan = await agent.create_plan("show me data distribution", "This is some dummy data.") - print("Plan:", plan) - except Exception as e: - print("Exception:", e) - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/scratch/test_env.py b/scratch/test_env.py deleted file mode 100644 index 1976bef..0000000 --- a/scratch/test_env.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -from dotenv import load_dotenv - -# Use absolute path to be sure -dotenv_path = r"e:\Agentloop\backend\.env" -print(f"Loading from: {dotenv_path}") -print(f"File exists: {os.path.exists(dotenv_path)}") - -success = load_dotenv(dotenv_path=dotenv_path) -print(f"Load success: {success}") - -print(f"SUPABASE_URL: {os.getenv('SUPABASE_URL')}") -print(f"SUPABASE_JWT_SECRET: {os.getenv('SUPABASE_JWT_SECRET')}") -print(f"SUPABASE_SERVICE_ROLE_KEY: {os.getenv('SUPABASE_SERVICE_ROLE_KEY')}") diff --git a/scratch/test_planner.py b/scratch/test_planner.py deleted file mode 100644 index c2b435f..0000000 --- a/scratch/test_planner.py +++ /dev/null @@ -1,16 +0,0 @@ -import asyncio -import sys -import os - -sys.path.insert(0, os.path.abspath('backend')) - -from core.planner.planner_agent import PlannerAgent - -async def main(): - agent = PlannerAgent() - print("Agent created") - plan = await agent.create_plan("show me data distribution", "This is some dummy data.") - print("Plan:", plan) - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/scratch/test_planner_debug.py b/scratch/test_planner_debug.py deleted file mode 100644 index 5bcc4a6..0000000 --- a/scratch/test_planner_debug.py +++ /dev/null @@ -1,26 +0,0 @@ -import asyncio -import sys -import os - -sys.path.insert(0, os.path.abspath('backend')) - -from core.planner.planner_agent import PlannerAgent -import logging - -logging.basicConfig(level=logging.DEBUG) - -async def main(): - agent = PlannerAgent() - print("Agent created") - - # We will step into get_chain - print("Calling planner create_plan") - - try: - plan = await asyncio.wait_for(agent.create_plan("show me data distribution", "This is some dummy data."), timeout=20.0) - print("Plan:", plan) - except asyncio.TimeoutError: - print("TIMED OUT!") - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/scratch/test_supabase_schema.py b/scratch/test_supabase_schema.py deleted file mode 100644 index 11284f3..0000000 --- a/scratch/test_supabase_schema.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import sys -from supabase import create_client - -# setup from .env -from dotenv import load_dotenv -load_dotenv('e:/Agentloop/backend/.env') - -url = os.environ.get("SUPABASE_URL") -key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY") - -client = create_client(url, key) - -print("Testing agent_runs...") -try: - res = client.table("agent_runs").select("*").limit(1).execute() - print("agent_runs schema:", res.data[0].keys() if res.data else "empty") -except Exception as e: - print("Error on agent_runs:", e) - -print("Testing uploaded_files...") -try: - res = client.table("uploaded_files").select("*").limit(1).execute() - print("uploaded_files schema:", res.data[0].keys() if res.data else "empty") -except Exception as e: - print("Error on uploaded_files:", e) diff --git a/src/App.jsx b/src/App.jsx index dba1c59..2549cdb 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -24,7 +24,7 @@ import { LandingPage } from './pages/LandingPage' import { ProjectsPage } from './pages/ProjectsPage' import { HomePage } from './pages/HomePage' import { ToastContainer } from './components/shared/Toast' -import { EvalDashboard } from './pages/EvalDashboard' + import { useFileUpload } from './hooks/useFileUpload' import { useAgentRun } from './hooks/useAgentRun' import './index.css' @@ -62,7 +62,7 @@ function ProtectedRoute({ children }) { * - Clean agent state per project */ function ProjectWorkspace() { - const { projectId } = useParams() + useParams() // projectId used as React key in ProjectWorkspaceRoute const fileState = useFileUpload() const agentState = useAgentRun(fileState.files) @@ -88,11 +88,7 @@ function AppInner() { } /> - - - - } /> + diff --git a/src/components/FileUploadPanel.jsx b/src/components/FileUploadPanel.jsx index 2dcee36..0f0cf60 100644 --- a/src/components/FileUploadPanel.jsx +++ b/src/components/FileUploadPanel.jsx @@ -1,51 +1,300 @@ -import React, { useCallback } from 'react' -import { UploadCloud, Trash2, FolderOpen } from 'lucide-react' +import React, { useCallback, useState } from 'react' +import { X, FileText, AlertCircle, Loader2, Trash2, ChevronDown, ChevronUp } from 'lucide-react' import { DropZone } from './DropZone' -import { FileList } from './FileList' -import { toast } from './Toast' +import { SchemaPreviewPanel } from './SchemaPreviewPanel' +import { useWorkspaceStore } from '../stores/workspaceStore' + +// File type → badge colour mapping +const FILE_TYPE_COLORS = { + csv: 'bg-emerald-100 text-emerald-700 border-emerald-200', + xlsx: 'bg-blue-100 text-blue-700 border-blue-200', + xls: 'bg-blue-100 text-blue-700 border-blue-200', + parquet: 'bg-violet-100 text-violet-700 border-violet-200', + json: 'bg-amber-100 text-amber-700 border-amber-200', + md: 'bg-slate-100 text-slate-700 border-slate-200', + pdf: 'bg-red-100 text-red-700 border-red-200', +} + +function getFileExt(filename) { + return filename?.split('.').pop()?.toLowerCase() ?? '' +} + +function formatRows(n) { + if (!n || n === 0) return null + return n >= 1000 ? `${(n / 1000).toFixed(1)}k rows` : `${n} rows` +} + +/** + * FileUploadPanel — multi-file upload panel with per-file chip management. + * + * Supports workspace-scoped batch uploads via POST /workspaces/{id}/upload. + * After each upload the returned MultiFileContext is stored in workspaceStore + * so SchemaPreviewPanel can render detected join candidates. + * + * Falls back to the legacy POST /upload endpoint when no activeWorkspace is set. + * + * @param {Object} props + * @param {Function} props.onUploaded - Called with array of accepted filenames after success. + * @param {Function} [props.onRejected] - Called with array of rejected file objects. + * @param {string} [props.sessionId] - Optional session identifier for the in-memory cache. + * @param {string} [props.apiBase] - Base URL for backend API calls (default: '/api'). + */ +export function FileUploadPanel({ + onUploaded, + onRejected, + sessionId, + apiBase = '/api', +}) { + const { activeWorkspace, setMultiFileContext, resetFileContext, multiFileContext } = + useWorkspaceStore() + + // Local state for file chips (separate from the store — these are ephemeral UI handles) + const [uploadedFiles, setUploadedFiles] = useState([]) + const [uploading, setUploading] = useState(false) + const [uploadErrors, setUploadErrors] = useState([]) + const [showSchema, setShowSchema] = useState(true) + + // ── Upload handler ───────────────────────────────────────────────────────── + + const handleFiles = useCallback(async (rawFiles) => { + if (!rawFiles?.length) return + + setUploading(true) + setUploadErrors([]) + + const formData = new FormData() + for (const f of rawFiles) formData.append('files', f) + if (sessionId) formData.append('session_id', sessionId) + + try { + let url + let response + let result + + if (activeWorkspace?.id) { + // Workspace-aware upload (Phase 9) + url = `${apiBase}/workspaces/${activeWorkspace.id}/upload` + if (sessionId) url += `?session_id=${encodeURIComponent(sessionId)}` + response = await fetch(url, { method: 'POST', body: formData }) + result = await response.json() + + if (!response.ok) { + throw new Error(result?.detail ?? `Upload failed (${response.status})`) + } + + // Update store with returned MultiFileContext + if (result.multi_file_context) { + setMultiFileContext(result.multi_file_context) + } + + const accepted = result.accepted_files ?? [] + const rejected = result.rejected_files ?? [] + + if (rejected.length && onRejected) { + onRejected(rejected.map((r) => ({ name: r.filename, reason: r.reason }))) + } + + if (accepted.length) { + const newChips = accepted.map((a) => ({ + filename: a.filename, + ext: getFileExt(a.filename), + // row count is embedded in the multi_file_context + rowCount: result.multi_file_context?.files?.find( + (f) => f.file_name === a.filename + )?.row_count ?? null, + })) + setUploadedFiles((prev) => [...prev, ...newChips]) + onUploaded?.(accepted.map((a) => a.filename)) + } + + if (rejected.length) { + setUploadErrors(rejected.map((r) => `${r.filename}: ${r.reason}`)) + } + } else { + // Legacy in-memory upload (backward compat) + url = `${apiBase}/upload` + response = await fetch(url, { method: 'POST', body: formData }) + result = await response.json() + + if (!response.ok) { + throw new Error(result?.detail ?? `Upload failed (${response.status})`) + } + + const accepted = result.accepted_files ?? [] + const rejected = result.rejected_files ?? [] + + if (rejected.length && onRejected) { + onRejected(rejected.map((r) => ({ name: r.filename, reason: r.reason }))) + } + + if (accepted.length) { + const newChips = accepted.map((a) => ({ + filename: a.filename, + ext: getFileExt(a.filename), + rowCount: null, + })) + setUploadedFiles((prev) => [...prev, ...newChips]) + onUploaded?.(accepted.map((a) => a.filename)) + } + + if (rejected.length) { + setUploadErrors(rejected.map((r) => `${r.filename}: ${r.reason}`)) + } + } + } catch (err) { + setUploadErrors([err.message ?? 'Upload failed.']) + } finally { + setUploading(false) + } + }, [activeWorkspace, sessionId, apiBase, onUploaded, onRejected, setMultiFileContext]) + + // ── Remove a single file ─────────────────────────────────────────────────── + + const handleRemoveFile = useCallback(async (chip) => { + if (!activeWorkspace?.id) { + // Legacy: just remove the chip (no DB row to delete) + setUploadedFiles((prev) => prev.filter((f) => f.filename !== chip.filename)) + return + } + + // Find the workspace_files id from multiFileContext + const fileEntry = multiFileContext?.files?.find((f) => f.file_name === chip.filename) + if (!fileEntry?.id) { + // Fallback: chip-only removal + setUploadedFiles((prev) => prev.filter((f) => f.filename !== chip.filename)) + return + } + + try { + const url = `${apiBase}/workspaces/${activeWorkspace.id}/files/${fileEntry.id}` + const response = await fetch(url, { method: 'DELETE' }) + const result = await response.json() + + if (!response.ok) throw new Error(result?.detail ?? 'Delete failed.') + + setUploadedFiles((prev) => prev.filter((f) => f.filename !== chip.filename)) + if (result.multi_file_context) { + setMultiFileContext(result.multi_file_context) + } + } catch (err) { + setUploadErrors([err.message ?? 'Delete failed.']) + } + }, [activeWorkspace, apiBase, multiFileContext, setMultiFileContext]) + + // ── Clear all files ──────────────────────────────────────────────────────── + + const handleClearAll = useCallback(() => { + setUploadedFiles([]) + resetFileContext() + setUploadErrors([]) + }, [resetFileContext]) + + // ── Render ───────────────────────────────────────────────────────────────── + + const hasFiles = uploadedFiles.length > 0 + const hasMultiContext = multiFileContext?.files?.length >= 2 + const hasCandidates = multiFileContext?.join_candidates?.length > 0 -export function FileUploadPanel({ files, onAddFiles, onRemoveFile, onClearAll }) { - const handleRejected = useCallback((rejectedFiles) => { - rejectedFiles.forEach((r) => toast(`"${r.name}" — ${r.reason}`, 'error')) - }, []) return ( -
- {/* Header */} -
-
-
-
- +
+ {/* Drop zone */} + + + {/* Upload spinner */} + {uploading && ( +
+ + Uploading files… +
+ )} + + {/* Error banner */} + {uploadErrors.length > 0 && ( +
+ {uploadErrors.map((err, i) => ( +
+ + {err}
-

Document Ingestion

+ ))} +
+ )} + + {/* File chip list */} + {hasFiles && ( +
+
+ + {uploadedFiles.length} file{uploadedFiles.length > 1 ? 's' : ''} uploaded + + {uploadedFiles.length > 1 && ( + + )} +
+ +
+ {uploadedFiles.map((chip) => { + const typeColor = FILE_TYPE_COLORS[chip.ext] ?? 'bg-slate-100 text-slate-600 border-slate-200' + const rows = formatRows(chip.rowCount) + + return ( +
+
+ + + {chip.filename} + +
+ +
+ {rows && ( + + {rows} + + )} + + {chip.ext} + + +
+
+ ) + })}
- {files.length > 0 && ( - - )}
-

- Supported: CSV, TXT, XLSX, PDF, JSON, Markdown, Parquet -

-
- - {/* Drop Zone */} - - - {/* File List */} - {files.length > 0 ? ( - - ) : ( -
- -

No files uploaded yet

+ )} + + {/* Schema preview (multi-file only) */} + {hasMultiContext && ( +
+ + {showSchema && }
)}
diff --git a/src/components/OutputPanel.jsx b/src/components/OutputPanel.jsx index c24b667..fc75e74 100644 --- a/src/components/OutputPanel.jsx +++ b/src/components/OutputPanel.jsx @@ -7,7 +7,7 @@ import { import { CodeBlock } from './CodeBlock' // Strip leading markdown/unicode bullet characters so CSS dot + embedded char don't double-up. -const cleanBullet = (s) => s.replace(/^[•\-\*]\s*/, '').trim() +const cleanBullet = (s) => s.replace(/^[•\-*]\s*/, '').trim() // ─── Status Panel ──────────────────────────────────────────────────────────── function StatusPanel({ status }) { diff --git a/src/components/SchemaPreviewPanel.jsx b/src/components/SchemaPreviewPanel.jsx new file mode 100644 index 0000000..baa420c --- /dev/null +++ b/src/components/SchemaPreviewPanel.jsx @@ -0,0 +1,332 @@ +import React, { useState, useCallback } from 'react' +import { + GitMerge, AlertCircle, CheckCircle2, HelpCircle, + ChevronDown, ChevronUp, Plus, X, +} from 'lucide-react' +import { useWorkspaceStore } from '../stores/workspaceStore' + +// --------------------------------------------------------------------------- +// Confidence dot +// --------------------------------------------------------------------------- + +function ConfidenceDot({ score }) { + const pct = Math.round(score * 100) + let color = 'bg-slate-300' + if (score >= 0.80) color = 'bg-emerald-400' + else if (score >= 0.50) color = 'bg-amber-400' + + return ( + + + {pct}% + + ) +} + +// --------------------------------------------------------------------------- +// File card +// --------------------------------------------------------------------------- + +const DTYPE_COLORS = { + int: 'bg-blue-50 text-blue-700', + float: 'bg-indigo-50 text-indigo-700', + object: 'bg-slate-50 text-slate-600', + bool: 'bg-amber-50 text-amber-700', +} + +function dtypeColor(dtype) { + const lower = dtype.toLowerCase() + if (lower.includes('int')) return DTYPE_COLORS.int + if (lower.includes('float')) return DTYPE_COLORS.float + if (lower.includes('bool')) return DTYPE_COLORS.bool + return DTYPE_COLORS.object +} + +function FileCard({ file }) { + const [expanded, setExpanded] = useState(false) + + return ( +
+ {/* Header */} +
+
+ + {file.var_name} + + + {file.file_name} + +
+
+ {file.row_count > 0 && ( + + {file.row_count >= 1000 + ? `${(file.row_count / 1000).toFixed(1)}k rows` + : `${file.row_count} rows`} + + )} + + {file.file_type} + + +
+
+ + {/* Column list */} + {expanded && file.columns?.length > 0 && ( +
+ {file.columns.map((col) => ( +
+ + {col.dtype} + + {col.name} + {col.sample_values?.length > 0 && ( + + {col.sample_values.slice(0, 2).join(', ')} + + )} +
+ ))} +
+ )} +
+ ) +} + +// --------------------------------------------------------------------------- +// Join override editor +// --------------------------------------------------------------------------- + +function JoinOverrideEditor({ files }) { + const { joinOverrides, addJoinOverride, removeJoinOverride } = useWorkspaceStore() + const [leftVar, setLeftVar] = useState(files[0]?.var_name ?? '') + const [leftCol, setLeftCol] = useState('') + const [rightVar, setRightVar] = useState(files[1]?.var_name ?? '') + const [rightCol, setRightCol] = useState('') + + const leftFile = files.find((f) => f.var_name === leftVar) + const rightFile = files.find((f) => f.var_name === rightVar) + + const handleAdd = useCallback(() => { + if (!leftVar || !leftCol || !rightVar || !rightCol) return + if (leftVar === rightVar) return + addJoinOverride({ left_var: leftVar, left_col: leftCol, right_var: rightVar, right_col: rightCol }) + setLeftCol('') + setRightCol('') + }, [leftVar, leftCol, rightVar, rightCol, addJoinOverride]) + + return ( +
+

+ Override join keys +

+ + {/* Existing overrides */} + {joinOverrides.length > 0 && ( +
+ {joinOverrides.map((o, i) => ( +
+ + {o.left_var}['{o.left_col}'] + {' '}↔{' '} + {o.right_var}['{o.right_col}'] + + +
+ ))} +
+ )} + + {/* Add override form */} +
+
+ + + +
+ + ↔ + +
+ + + +
+ + +
+
+ ) +} + +// --------------------------------------------------------------------------- +// Main export +// --------------------------------------------------------------------------- + +/** + * SchemaPreviewPanel — displays the multi-file context returned by the backend. + * + * Shows: + * - Horizontal scrollable row of file cards (variable name, filename, row count, columns). + * - "Detected join keys" section with coloured confidence indicators. + * - Expandable "Override join keys" section for manual control. + * + * @param {Object} props + * @param {import('../stores/workspaceStore').MultiFileContext} props.context + */ +export function SchemaPreviewPanel({ context }) { + const [showOverrideEditor, setShowOverrideEditor] = useState(false) + + if (!context?.files?.length) return null + + const { files, join_candidates: candidates = [] } = context + + return ( +
+ {/* File cards */} +
+

+ Uploaded files +

+
+ {files.map((file) => ( + + ))} +
+
+ + {/* Join candidates */} +
+
+ +

+ Detected join keys +

+
+ + {candidates.length === 0 ? ( +
+ + No join keys detected automatically. Use the override editor below. +
+ ) : ( +
+ {candidates.map((c, i) => ( +
+
+ {c.confidence >= 0.80 ? ( + + ) : c.confidence >= 0.50 ? ( + + ) : ( + + )} + + {c.left_var} + ['{c.left_col}'] + {' '}↔{' '} + {c.right_var} + ['{c.right_col}'] + +
+
+ + + {c.reason} + +
+
+ ))} +
+ )} +
+ + {/* Override editor toggle */} +
+ + + {showOverrideEditor && ( +
+ +
+ )} +
+
+ ) +} diff --git a/src/components/agent/AgentMetricsOverlay.jsx b/src/components/agent/AgentMetricsOverlay.jsx deleted file mode 100644 index c643c9e..0000000 --- a/src/components/agent/AgentMetricsOverlay.jsx +++ /dev/null @@ -1,178 +0,0 @@ -/** - * AgentMetricsOverlay.jsx - * - * Developer/audit diagnostic panel that aggregates evaluation quality metrics - * across DS-STAR agent runs. Surfaces the same convergence data used in the - * paper's ablation studies (easy vs. hard task round distribution). - * - * Props: - * - metrics: RunMetrics summary dict from the "metrics" SSE event payload. - * - totalRunMs: number — wall-clock ms for the full run. - * - complexity: "easy" | "hard" - * - rounds: number — rounds completed. - * - isVisible: boolean — controlled by parent (e.g. a hotkey or dev toggle). - */ - -import React, { useState } from 'react' -import { - BarChart2, Zap, Clock, ChevronDown, ChevronUp, - Activity, GitMerge, CheckCircle2, XCircle, -} from 'lucide-react' - -// ─── helpers ───────────────────────────────────────────────────────────────── - -function ms(val) { - if (!val || val <= 0) return '—' - if (val < 1000) return `${val}ms` - return `${(val / 1000).toFixed(1)}s` -} - -function pct(part, total) { - if (!total || total <= 0) return '0%' - return `${Math.round((part / total) * 100)}%` -} - -// ─── Stage bar ──────────────────────────────────────────────────────────────── -function StageBar({ label, value, total, color }) { - const width = total > 0 ? Math.max(2, Math.round((value / total) * 100)) : 0 - return ( -
- {label} -
-
-
- - {ms(value)} {pct(value, total)} - -
- ) -} - -// ─── Round row ──────────────────────────────────────────────────────────────── -function RoundRow({ round }) { - return ( -
-
- Round {round.round_num} -
- {round.exec_success - ? - : } - - {round.is_sufficient ? 'Approved ✓' : `${Math.round(round.verifier_confidence * 100)}% conf`} - - {ms(round.total_ms)} -
-
-
- {round.round_num === 1 && round.planner_ms > 0 && ( - - )} - - - - {round.router_ms > 0 && ( - - )} -
-
- ) -} - -// ─── Main Component ─────────────────────────────────────────────────────────── -export function AgentMetricsOverlay({ metrics, totalRunMs, complexity, rounds, isVisible }) { - const [collapsed, setCollapsed] = useState(false) - const [roundsCollapsed, setRoundsCollapsed] = useState(true) - - if (!isVisible || !metrics) return null - - const perRound = metrics.per_round || [] - const routerFix = perRound.filter(r => r.router_ms > 0).length - const failedRounds = perRound.filter(r => !r.is_sufficient).length - - return ( -
- {/* Header */} - - - {!collapsed && ( -
- - {/* KPI row */} -
- {[ - { icon: Clock, label: 'Total Time', value: ms(totalRunMs), color: 'text-violet-600' }, - { icon: Activity, label: 'Rounds Used', value: `${rounds}/${metrics.rounds_completed ?? rounds}`, color: 'text-blue-600' }, - { icon: GitMerge, label: 'Router Fixes', value: routerFix, color: 'text-amber-600' }, - { icon: CheckCircle2, label: 'Fail / Pass', value: `${failedRounds} / ${rounds - failedRounds}`, color: 'text-emerald-600' }, - ].map(({ icon: Icon, label, value, color }) => ( -
- - {value} - {label} -
- ))} -
- - {/* Per-round breakdown toggle */} -
- - - {!roundsCollapsed && perRound.length > 0 && ( -
- {perRound.map(r => )} -
- )} -
- -

- Paper benchmark — Easy tasks: ~3.0 rounds avg · Hard tasks: ~5.6 rounds avg (DABStep) -

-
- )} -
- ) -} diff --git a/src/components/agent/AgentProgressPanel.jsx b/src/components/agent/AgentProgressPanel.jsx index b4903f6..5d5a376 100644 --- a/src/components/agent/AgentProgressPanel.jsx +++ b/src/components/agent/AgentProgressPanel.jsx @@ -9,11 +9,10 @@ import { useAuth } from '../../contexts/AuthContext' import { toast } from '../shared/Toast' import { PlanStepList } from './PlanStepList' import { CodeBlock } from './CodeBlock' -import { AgentMetricsOverlay } from './AgentMetricsOverlay' // Strip leading markdown/unicode bullet characters from a string so we don't // render a CSS dot AND an embedded '• ' or '* ' simultaneously. -const cleanBullet = (s) => s.replace(/^[•\-\*]\s*/, '').trim() +const cleanBullet = (s) => s.replace(/^[•\-*]\s*/, '').trim() // ─── Phase meta ───────────────────────────────────────────────────────────── const PHASE_META = { @@ -351,11 +350,6 @@ export function AgentProgressPanel({ activeRunId, onReset, onRerun, - // Metrics props (from 'metrics' SSE event) - runMetrics, - totalRunMs, - complexity, - showMetrics, // Deep Research mode props isResearchMode, subQuestions, @@ -685,15 +679,6 @@ export function AgentProgressPanel({
)} - {/* ── Evaluation Metrics Overlay (when run completes) ──────────── */} - - {/* ── Idle empty state ─────────────────────────────────────────── */} {agentStatus === 'idle' && (
· {run.rounds} round(s) )} - {run.eval_metrics?.total_run_ms > 0 && ( - - · {Math.round(run.eval_metrics.total_run_ms / 1000)}s - - )} -
diff --git a/src/components/agent/agentSettingsConstants.js b/src/components/agent/agentSettingsConstants.js new file mode 100644 index 0000000..caef4b2 --- /dev/null +++ b/src/components/agent/agentSettingsConstants.js @@ -0,0 +1,13 @@ +/** + * agentSettingsConstants.js — Shared constants for agent settings. + * + * Extracted from AgentSettings.jsx so that the component file only exports + * React components (required for Fast Refresh to work correctly). + */ + +export const DEFAULT_SETTINGS = { + maxRounds: 10, + model: 'meta/llama-3.1-70b-instruct', + coderModel: 'meta/codellama-70b-instruct', + temperature: 0.1, +} diff --git a/src/components/eval/AgentTable.jsx b/src/components/eval/AgentTable.jsx deleted file mode 100644 index 2fca7d7..0000000 --- a/src/components/eval/AgentTable.jsx +++ /dev/null @@ -1,49 +0,0 @@ -/** - * AgentTable.jsx - * Displays a table of per-agent performance metrics. - */ - -import React from 'react' - -export function AgentTable({ agents = [] }) { - if (agents.length === 0) { - return
No agent metrics available.
- } - - return ( -
- - - - - - - - - - - {agents.map((agent) => ( - - - - - - - ))} - -
AgentAvg LatencyFailure RateTotal Calls
- {agent.agent_name} - - {agent.avg_latency_ms} ms - - 0.05 ? 'bg-rose-50 text-rose-700' : 'bg-emerald-50 text-emerald-700' - }`}> - {(agent.failure_rate * 100).toFixed(1)}% - - - {agent.total_calls} -
-
- ) -} diff --git a/src/components/eval/DebugChart.jsx b/src/components/eval/DebugChart.jsx deleted file mode 100644 index e0a904c..0000000 --- a/src/components/eval/DebugChart.jsx +++ /dev/null @@ -1,63 +0,0 @@ -/** - * DebugChart.jsx - * Displays debug loop analysis with charts. - */ - -import React from 'react' -import { PieChart, Pie, Cell, ResponsiveContainer, Tooltip, Legend, BarChart, Bar, XAxis, YAxis, CartesianGrid } from 'recharts' - -const COLORS = ['#8b5cf6', '#ec4899', '#f59e0b', '#10b981', '#3b82f6', '#6366f1'] - -export function DebugChart({ debugStats }) { - if (!debugStats) return null - - const data = debugStats.error_type_distribution || [] - - return ( -
-

Debugger Error Types

- - {data.length === 0 ? ( -
- No error data available -
- ) : ( -
- - - - {data.map((entry, index) => ( - - ))} - - - - - -
- )} - -
-
-

{debugStats.avg_debug_depth}

-

Avg Debug Depth

-
-
-

{(debugStats.retry_success_ratio * 100).toFixed(1)}%

-

Retry Recovery Rate

-
-
-
- ) -} diff --git a/src/components/eval/KpiCard.jsx b/src/components/eval/KpiCard.jsx deleted file mode 100644 index f330fbd..0000000 --- a/src/components/eval/KpiCard.jsx +++ /dev/null @@ -1,29 +0,0 @@ -/** - * KpiCard.jsx — stat tile with icon, value, sub-label, and optional trend. - * - * Props: - * icon — Lucide React icon component - * label — metric label - * value — formatted string value - * sub — optional sub-label beneath value - * color — Tailwind text-* class for the icon (default: text-violet-500) - * bg — Tailwind bg-* class for the icon background (default: bg-violet-50) - * border — Tailwind border-* for icon ring (default: border-violet-100) - */ - -import React from 'react' - -export function KpiCard({ icon: Icon, label, value, sub, color = 'text-violet-500', bg = 'bg-violet-50', border = 'border-violet-100' }) { - return ( -
-
- -
-
-

{value}

-

{label}

- {sub &&

{sub}

} -
-
- ) -} diff --git a/src/components/eval/RunList.jsx b/src/components/eval/RunList.jsx deleted file mode 100644 index 5031da4..0000000 --- a/src/components/eval/RunList.jsx +++ /dev/null @@ -1,95 +0,0 @@ -/** - * RunList.jsx - * Filterable run list. - */ - -import React, { useState } from 'react' -import { Search, Filter, CheckCircle2, XCircle } from 'lucide-react' - -export function RunList({ runs = [], onSelectRun, selectedRunId }) { - const [diffFilter, setDiffFilter] = useState('all') - const [statusFilter, setStatusFilter] = useState('all') - - const filteredRuns = runs.filter(run => { - if (diffFilter !== 'all' && run.difficulty !== diffFilter) return false - if (statusFilter !== 'all') { - const isSuccess = run.success_rate >= 1.0 - if (statusFilter === 'success' && !isSuccess) return false - if (statusFilter === 'failed' && isSuccess) return false - } - return true - }) - - return ( -
- {/* Header and Filters */} -
-

Evaluation Runs

-
- - -
-
- - {/* List */} -
- {filteredRuns.length === 0 ? ( -
No runs found matching filters.
- ) : ( -
- {filteredRuns.map(run => { - const isSelected = selectedRunId === run.run_id - const isSuccess = run.success_rate >= 1.0 - const queryStr = run.agent_runs?.query || 'Unknown query' - - return ( - - ) - })} -
- )} -
-
- ) -} diff --git a/src/components/eval/TracePanel.jsx b/src/components/eval/TracePanel.jsx deleted file mode 100644 index b2bc465..0000000 --- a/src/components/eval/TracePanel.jsx +++ /dev/null @@ -1,110 +0,0 @@ -/** - * TracePanel.jsx - * Displays step-by-step trace of a single run. - */ - -import React from 'react' -import { Code2, Settings, AlertTriangle, FileCheck, CheckCircle2, XCircle, ArrowRightCircle } from 'lucide-react' - -const AGENT_ICONS = { - Analyzer: FileCheck, - Planner: Settings, - Coder: Code2, - Executor: ArrowRightCircle, - Debugger: AlertTriangle, - Verifier: CheckCircle2, - Router: Settings, - Finalizer: FileCheck, -} - -export function TracePanel({ traceData }) { - if (!traceData) { - return ( -
-

Select a run from the list to view its execution trace.

-
- ) - } - - const { run, steps } = traceData - - return ( -
-
-

Execution Trace

-

Run ID: {traceData.run_id.substring(0, 8)}...

- {run?.query && ( -
- "{run.query}" -
- )} -
- -
- {(!steps || steps.length === 0) ? ( -

No steps recorded for this run.

- ) : ( -
- {steps.map((step, idx) => { - const Icon = AGENT_ICONS[step.agent_name] || CheckCircle2 - const isError = step.error_type || !step.validation_passed - - return ( -
- {/* Timeline column */} -
-
- -
- {idx < steps.length - 1 &&
} -
- - {/* Content column */} -
-
-
- {step.agent_name} - - R{step.round_num} - - - {step.latency_ms > 0 ? `${step.latency_ms}ms` : ''} - -
- - {step.error_type && ( -
- {step.error_type} -
- )} - - {step.input_summary && ( -
- Input -

- {step.input_summary} -

-
- )} - - {step.output_summary && ( -
- Output -

- {step.output_summary} -

-
- )} -
-
-
- ) - })} -
- )} -
-
- ) -} diff --git a/src/components/shared/WorkspaceSelector.jsx b/src/components/shared/WorkspaceSelector.jsx index b47f27e..09bec5e 100644 --- a/src/components/shared/WorkspaceSelector.jsx +++ b/src/components/shared/WorkspaceSelector.jsx @@ -65,7 +65,7 @@ export function WorkspaceSelector() { setOpen(false) }, [setActiveWorkspace]) - const handleCreate = useCallback(async (e) => { + const handleCreate = async (e) => { e.preventDefault() if (!newName.trim() || !user?.id) return setCreateLoading(true) @@ -74,7 +74,7 @@ export function WorkspaceSelector() { setCreating(false) setCreateLoading(false) setOpen(false) - }, [newName, user?.id, createWorkspace]) + } // Don't render for anonymous users if (!isAuthenticated) return null diff --git a/src/components/upload/FileUploadPanel.jsx b/src/components/upload/FileUploadPanel.jsx index cb2d4cf..d439362 100644 --- a/src/components/upload/FileUploadPanel.jsx +++ b/src/components/upload/FileUploadPanel.jsx @@ -1,11 +1,11 @@ import React, { useCallback } from 'react' -import { UploadCloud, Trash2, FolderOpen, FolderKanban } from 'lucide-react' +import { UploadCloud, Trash2, FolderOpen, FolderKanban, Loader2 } from 'lucide-react' import { DropZone } from './DropZone' import { FileList } from './FileList' import { toast } from '../shared/Toast' import { useWorkspaceStore } from '../../stores/workspaceStore' -export function FileUploadPanel({ files, onAddFiles, onRemoveFile, onClearAll }) { +export function FileUploadPanel({ files, filesLoading, onAddFiles, onRemoveFile, onClearAll }) { const handleRejected = useCallback((rejectedFiles) => { rejectedFiles.forEach((r) => toast(`"${r.name}" — ${r.reason}`, 'error')) }, []) @@ -57,6 +57,11 @@ export function FileUploadPanel({ files, onAddFiles, onRemoveFile, onClearAll }) {/* File List */} {files.length > 0 ? ( + ) : filesLoading ? ( +
+ +

Loading files…

+
) : (
diff --git a/src/contexts/AuthContext.jsx b/src/contexts/AuthContext.jsx index 298c474..a109076 100644 --- a/src/contexts/AuthContext.jsx +++ b/src/contexts/AuthContext.jsx @@ -12,6 +12,8 @@ * the singleton supabaseClient which reads from VITE_* env vars. */ +/* eslint-disable react-refresh/only-export-components */ + import React, { createContext, useContext, diff --git a/src/hooks/useAgentRun.js b/src/hooks/useAgentRun.js index d274016..50d7d0a 100644 --- a/src/hooks/useAgentRun.js +++ b/src/hooks/useAgentRun.js @@ -12,7 +12,7 @@ import { useState, useCallback, useRef } from 'react' import { toast } from '../components/shared/Toast' import { createCancellableStream } from '../services/agentApi' -import { DEFAULT_SETTINGS } from '../components/agent/AgentSettings' +import { DEFAULT_SETTINGS } from '../components/agent/agentSettingsConstants' import { useAuth } from '../contexts/AuthContext' import { useWorkspaceStore } from '../stores/workspaceStore' @@ -41,11 +41,6 @@ export function useAgentRun(files) { const [historyLoading, setHistoryLoading] = useState(false) const [settings, setSettings] = useState({ ...DEFAULT_SETTINGS }) const [activeRunId, setActiveRunId] = useState(null) - // Evaluation metrics (from 'metrics' SSE event) - const [runMetrics, setRunMetrics] = useState(null) - const [totalRunMs, setTotalRunMs] = useState(0) - const [complexity, setComplexity] = useState('easy') - const [showMetrics, setShowMetrics] = useState(false) const [lastSubmittedQuery, setLastSubmittedQuery] = useState('') // Deep Research mode state @@ -354,17 +349,6 @@ export function useAgentRun(files) { setTimeout(() => fetchHistory(20, true), 1500) break - case 'metrics': - setRunMetrics(payload.metrics || null) - setTotalRunMs(payload.total_run_ms || 0) - setComplexity(payload.complexity || 'easy') - setShowMetrics(true) - addLog( - `[Metrics] Task complexity: ${payload.complexity || 'easy'} · Total: ${Math.round((payload.total_run_ms || 0) / 1000)}s`, - 'info', - ) - break - case 'warning': addLog('[⚠] ' + payload.message, 'warn') break @@ -424,10 +408,6 @@ export function useAgentRun(files) { setOutput(null) setVerifierFeedback(null) setActiveRunId(null) - setRunMetrics(null) - setTotalRunMs(0) - setComplexity('easy') - setShowMetrics(false) setIsResearchMode(false) setSubQuestions([]) setResearchReport(null) @@ -497,10 +477,6 @@ export function useAgentRun(files) { setOutput(null) setVerifierFeedback(null) setActiveRunId(null) - setRunMetrics(null) - setTotalRunMs(0) - setComplexity('easy') - setShowMetrics(false) setIsResearchMode(false) setSubQuestions([]) setResearchReport(null) @@ -565,11 +541,6 @@ export function useAgentRun(files) { handleReset, fetchHistory, loadRun, - // Evaluation metrics - runMetrics, - totalRunMs, - complexity, - showMetrics, lastSubmittedQuery, // Deep Research state isResearchMode, diff --git a/src/hooks/useFileUpload.js b/src/hooks/useFileUpload.js index 0348b91..8b31386 100644 --- a/src/hooks/useFileUpload.js +++ b/src/hooks/useFileUpload.js @@ -18,6 +18,27 @@ import { useWorkspaceStore } from '../stores/workspaceStore' let fileIdCounter = 0 +/** + * Maps a pandas dtype string to a user-friendly display label. + * e.g. "int64" → "INTEGER", "float64" → "FLOAT", "object" → "STRING" + * + * @param {string} dtype - Raw pandas dtype string + * @returns {string} User-friendly type label + */ +const mapPandasDtype = (dtype) => { + if (!dtype || typeof dtype !== 'string') return 'STRING' + const d = dtype.toLowerCase() + if (d.startsWith('int') || d === 'int64' || d === 'int32' || d === 'int16' || d === 'int8') return 'INTEGER' + if (d.startsWith('uint')) return 'INTEGER' + if (d.startsWith('float') || d === 'float64' || d === 'float32' || d === 'float16') return 'FLOAT' + if (d === 'bool' || d === 'boolean' || d.startsWith('bool')) return 'BOOLEAN' + if (d.startsWith('datetime') || d.includes('datetime')) return 'DATETIME' + if (d.startsWith('timedelta')) return 'TIMEDELTA' + if (d === 'category') return 'CATEGORY' + if (d === 'string' || d === 'object') return 'STRING' + return dtype.toUpperCase() +} + /** * Generates a stable session ID for this browser tab. * Uses crypto.randomUUID when available, falls back to a random hex string. @@ -94,10 +115,11 @@ const extractColumns = async (fileObj, filename, url = null) => { export function useFileUpload() { const [files, setFiles] = useState([]) const [pendingDuplicates, setPendingDuplicates] = useState([]) + const [filesLoading, setFilesLoading] = useState(false) // Auth token for API calls — reads live from AuthContext const { getAccessToken } = useAuth() - const { activeWorkspace } = useWorkspaceStore() + const { activeWorkspace, setMultiFileContext } = useWorkspaceStore() // Derive sessionId from the active projectId so that the backend file-cache // bucket is scoped per project. Falls back to a random UUID when no project @@ -116,21 +138,37 @@ export function useFileUpload() { const fetchUploadedFiles = useCallback(async () => { if (!activeWorkspace?.id) return const token = getAccessToken() + setFilesLoading(true) try { - const res = await fetch(`/api/files?workspace_id=${activeWorkspace.id}`, { + // Use workspace-scoped endpoint which reads from workspace_files table. + // This is where workspace uploads (POST /workspaces/{id}/upload) save data. + // The old /api/files endpoint reads from uploaded_files table which is + // the wrong source for workspace-scoped files. + const res = await fetch(`/api/workspaces/${activeWorkspace.id}/files`, { headers: token ? { Authorization: `Bearer ${token}` } : {}, }) if (res.ok) { const data = await res.json() const formattedFiles = await Promise.all(data.map(async (r, i) => { - const columns = await extractColumns(null, r.filename, r.file_url) + const columns = r.schema_json?.columns + ? r.schema_json.columns.map(c => { + // c can be a plain string (column name) or an object with .name/.dtype + const colName = typeof c === 'string' ? c : (c.name || c) + // Dtypes live in schema_json.dtypes as a separate dict: {"col": "int64", ...} + const rawDtype = (typeof c === 'object' && (c.dtype || c.type)) + || (r.schema_json.dtypes && r.schema_json.dtypes[colName]) + || 'string' + return { name: colName, type: mapPandasDtype(rawDtype) } + }) + : await extractColumns(null, r.filename, r.file_url || null) return { id: 10000 + i, + dbId: r.id, name: r.filename, size: r.file_size, progress: 100, _raw: null, - url: r.file_url, + url: r.file_url || null, metadata: columns ? { columns } : null } })) @@ -138,6 +176,8 @@ export function useFileUpload() { } } catch (err) { console.error(err) + } finally { + setFilesLoading(false) } }, [activeWorkspace?.id, getAccessToken]) @@ -172,12 +212,16 @@ export function useFileUpload() { `${acceptedNames.length} file${acceptedNames.length > 1 ? 's' : ''} uploaded successfully`, 'success' ) + // Store multiFileContext from workspace upload response for join inference + if (uploadResult.multi_file_context) { + setMultiFileContext(uploadResult.multi_file_context) + } } catch (err) { toast(`Processing error: ${err.message}`, 'error') } } }, - [applyProgress, getAccessToken, sessionId, activeWorkspace?.id] + [applyProgress, getAccessToken, sessionId, activeWorkspace?.id, setMultiFileContext] ) const handleAddFiles = useCallback( @@ -254,6 +298,7 @@ export function useFileUpload() { return { files, + filesLoading, pendingDuplicates, sessionId, handleAddFiles, diff --git a/src/pages/EvalDashboard.jsx b/src/pages/EvalDashboard.jsx deleted file mode 100644 index 264f475..0000000 --- a/src/pages/EvalDashboard.jsx +++ /dev/null @@ -1,444 +0,0 @@ -import React, { useState, useEffect, useRef } from 'react' -import { Link, useNavigate } from 'react-router-dom' -import { ThemeToggle } from '../components/shared/ThemeToggle' -import { useAuth } from '../contexts/AuthContext' -import { KpiCard } from '../components/eval/KpiCard' -import { AgentTable } from '../components/eval/AgentTable' -import { RunList } from '../components/eval/RunList' -import { TracePanel } from '../components/eval/TracePanel' -import { DebugChart } from '../components/eval/DebugChart' -import { - getOverview, - getAgentPerformance, - getDebugLoopStats, - listEvalRuns, - getRunTrace, -} from '../services/evalApi' -import { - Activity, - Clock, - RefreshCw, - CheckCircle2, - ArrowLeft, - BarChart2, - Cpu, - Bug, - Search, -} from 'lucide-react' -import { - LineChart, - Line, - XAxis, - YAxis, - CartesianGrid, - Tooltip, - ResponsiveContainer, - BarChart, - Bar, - Cell, - PieChart, - Pie, - Legend, - RadarChart, - Radar, - PolarGrid, - PolarAngleAxis, - PolarRadiusAxis, - AreaChart, - Area, -} from 'recharts' - -const BRAND_COLORS = ['#7c3aed', '#ec4899', '#f59e0b', '#10b981', '#3b82f6', '#6366f1', '#ef4444', '#14b8a6'] - -const NAV_SECTIONS = [ - { id: 'overview', label: 'System Overview', icon: Activity }, - { id: 'performance', label: 'Agent Performance', icon: Cpu }, - { id: 'debug', label: 'Debug Loop', icon: Bug }, - { id: 'explorer', label: 'Run Explorer', icon: Search }, -] - -export function EvalDashboard() { - const [overview, setOverview] = useState(null) - const [agents, setAgents] = useState([]) - const [debugStats, setDebugStats] = useState(null) - const [runs, setRuns] = useState([]) - const [selectedRunId, setSelectedRunId] = useState(null) - const [traceData, setTraceData] = useState(null) - const [loading, setLoading] = useState(true) - const [activeSection, setActiveSection] = useState('overview') - - const { isAuthenticated, loading: authLoading, getAccessToken } = useAuth() - const navigate = useNavigate() - - useEffect(() => { - if (!authLoading && !isAuthenticated) { - navigate('/', { replace: true }) - } - }, [authLoading, isAuthenticated, navigate]) - - const sectionRefs = { - overview: useRef(null), - performance: useRef(null), - debug: useRef(null), - explorer: useRef(null), - } - - /* ── Initial data load ── */ - useEffect(() => { - async function load() { - try { - setLoading(true) - const token = getAccessToken() - const [ov, ag, ds, rl] = await Promise.all([ - getOverview(token), - getAgentPerformance(token), - getDebugLoopStats(token), - listEvalRuns({}, token), - ]) - setOverview(ov) - setAgents(ag) - setDebugStats(ds) - setRuns(rl) - } catch (err) { - console.error('Dashboard load error', err) - } finally { - setLoading(false) - } - } - load() - }, [getAccessToken]) - - /* ── Trace load when run selected ── */ - useEffect(() => { - if (!selectedRunId) { setTraceData(null); return } - const token = getAccessToken() - getRunTrace(selectedRunId, token) - .then(setTraceData) - .catch(() => setTraceData(null)) - }, [selectedRunId, getAccessToken]) - - /* ── IntersectionObserver to track active section ── */ - useEffect(() => { - const observers = [] - Object.entries(sectionRefs).forEach(([id, ref]) => { - if (!ref.current) return - const obs = new IntersectionObserver( - ([entry]) => { if (entry.isIntersecting) setActiveSection(id) }, - { threshold: 0.3 } - ) - obs.observe(ref.current) - observers.push(obs) - }) - return () => observers.forEach(o => o.disconnect()) - }, [loading]) - - const scrollTo = (id) => { - sectionRefs[id]?.current?.scrollIntoView({ behavior: 'smooth', block: 'start' }) - } - - /* ── Derived chart data ── */ - const latencyChartData = agents.map(a => ({ - name: a.agent_name, - latency: Math.round(a.avg_latency_ms), - calls: a.total_calls, - })) - - const rawTrendData = runs.slice(0, 15).reverse().map((r, i) => ({ - idx: i + 1, - success: r.success_rate ?? 0, - time: new Date(r.created_at).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }), - })) - - // Recharts doesn't render lines/areas with a single data point. Pad it to a flat line. - const successTrendData = rawTrendData.length === 1 - ? [{...rawTrendData[0], time: 'Start'}, rawTrendData[0]] - : rawTrendData - - const radarData = agents.map(a => ({ - agent: a.agent_name, - reliability: Math.max(0, 1 - a.failure_rate) * 100, - throughput: Math.min(100, (a.total_calls / Math.max(1, Math.max(...agents.map(x => x.total_calls)))) * 100), - })) - - if (loading && !overview) { - return ( -
-
- -

Loading observability data…

-
-
- ) - } - - return ( -
- - {/* ── Sticky Header ── */} -
-
-
- - - Back - -
-
-

Agent Observability

-

Evaluation metrics, error distributions & step-by-step traces

-
-
- - {/* Section nav pills */} -
- - -
-
-
- - {/* ── Scrollable Content ── */} -
- - {/* ══════════ SECTION 1: System Overview ══════════ */} -
- - - {/* KPI row */} - {overview && ( -
- - - - -
- )} - - {/* Success trend line chart */} - - - - - - - - - - - - `${(v * 100).toFixed(0)}%`} domain={[0, 1]} tick={{ fontSize: 10, fill: '#94a3b8' }} /> - [`${(v * 100).toFixed(1)}%`, 'Success Rate']} - contentStyle={{ borderRadius: '10px', border: 'none', boxShadow: '0 4px 16px rgba(0,0,0,0.08)', fontSize: 12 }} /> - - - - - - {/* Runs per time bar chart */} - {runs.length > 0 && ( - - - r.difficulty !== 'hard').length }, - { label: 'Hard', count: runs.filter(r => r.difficulty === 'hard').length }, - ]} barSize={48}> - - - - - - - - - - - - )} -
- - - - {/* ══════════ SECTION 2: Agent Performance ══════════ */} -
- - - {/* Per-agent metrics table */} - - - - - {/* Latency bar chart */} - - {latencyChartData.length > 0 ? ( - - - - - `${(v / 1000).toFixed(1)}s`} tick={{ fontSize: 10, fill: '#94a3b8' }} /> - [`${v} ms`, 'Avg Latency']} - contentStyle={{ borderRadius: '10px', border: 'none', boxShadow: '0 4px 16px rgba(0,0,0,0.08)', fontSize: 12 }} /> - - {latencyChartData.map((_, i) => ( - - ))} - - - - ) : ( -
- No latency data available. -
- )} - - - {/* Radar chart */} - - {radarData.length > 0 ? ( - - - - - - - - - - - - ) : ( -
- No reliability data available. -
- )} - -
- - - - {/* ══════════ SECTION 3: Debug Loop ══════════ */} -
- - - {debugStats ? ( - <> - {/* Debug stats KPIs */} -
-
-

{debugStats.avg_debug_depth}

-

Avg Debug Depth

-
-
-

- {(debugStats.retry_success_ratio * 100).toFixed(1)}% -

-

Retry Recovery Rate

-
-
- - {/* Debug chart (pie + stats) */} - - - {/* Error type bar chart (if data exists) */} - {(debugStats.error_type_distribution || []).length > 0 && ( - - - - - - - - - {(debugStats.error_type_distribution).map((_, i) => ( - - ))} - - - - - )} - - ) : ( -
- No debug data available. -
- )} -
- - - - {/* ══════════ SECTION 4: Run Explorer ══════════ */} -
- - -
-
- -
-
- -
-
-
- - {/* Bottom padding */} -
-
-
- ) -} - -/* ── Helpers ── */ - -function SectionTitle({ icon: Icon, title, subtitle }) { - return ( -
-
- -
-
-

{title}

- {subtitle &&

{subtitle}

} -
-
- ) -} - -function ChartCard({ title, subtitle, children, className = '' }) { - return ( -
- {(title || subtitle) && ( -
- {title &&

{title}

} - {subtitle &&

{subtitle}

} -
- )} - {children} -
- ) -} - -function Divider() { - return
-} diff --git a/src/pages/HomePage.jsx b/src/pages/HomePage.jsx index 2767090..0503f78 100644 --- a/src/pages/HomePage.jsx +++ b/src/pages/HomePage.jsx @@ -18,6 +18,7 @@ export function HomePage({ fileState, agentState }) { const { files, + filesLoading, pendingDuplicates, sessionId, handleAddFiles, @@ -48,10 +49,6 @@ export function HomePage({ fileState, agentState }) { handleReset, fetchHistory, loadRun, - runMetrics, - totalRunMs, - complexity, - showMetrics, } = agentState const { user, isAuthenticated, signOut, loading: authLoading } = useAuth() @@ -126,14 +123,6 @@ export function HomePage({ fileState, agentState }) {
- - - rerunLastQuery(sessionId)} - runMetrics={runMetrics} - totalRunMs={totalRunMs} - complexity={complexity} - showMetrics={showMetrics} />
diff --git a/src/pages/ProjectsPage.jsx b/src/pages/ProjectsPage.jsx index cc27a6a..43d620e 100644 --- a/src/pages/ProjectsPage.jsx +++ b/src/pages/ProjectsPage.jsx @@ -10,7 +10,7 @@ import { useWorkspaceStore } from '../stores/workspaceStore' import { supabase } from '../lib/supabaseClient' export function ProjectsPage() { - const { user, signOut, isAuthenticated, loading: authLoading, getAccessToken } = useAuth() + const { user, signOut, isAuthenticated, loading: authLoading } = useAuth() const { workspaces, loading: wsLoading, fetchWorkspaces, createWorkspace, setActiveWorkspace } = useWorkspaceStore() const navigate = useNavigate() @@ -23,7 +23,7 @@ export function ProjectsPage() { const [searchQuery, setSearchQuery] = useState('') const [sortConfig, setSortConfig] = useState({ key: 'created_at', direction: 'desc' }) const [currentPage, setCurrentPage] = useState(1) - const itemsPerPage = 10 + const itemsPerPage = 5 // Filtering and Sorting logic const filteredAndSortedWorkspaces = useMemo(() => { @@ -148,7 +148,7 @@ export function ProjectsPage() { } return ( -
+
{/* Navbar */}
@@ -178,7 +178,8 @@ export function ProjectsPage() {
{/* Main Content */} -
+
+

Your Projects

@@ -257,7 +258,7 @@ export function ProjectsPage() { ) : ( !isCreating && (
-
+
@@ -369,7 +370,16 @@ export function ProjectsPage() { ) )} + + + {/* Footer */} +
+
+ © {new Date().getFullYear()} Agentloop. All rights reserved. + Intelligent Document Processing +
+
) } diff --git a/src/services/api.js b/src/services/api.js index ec6be17..e02afc6 100644 --- a/src/services/api.js +++ b/src/services/api.js @@ -61,10 +61,19 @@ export async function uploadFiles(files, onProgress, sessionId = '', accessToken const formData = new FormData() files.forEach((file) => formData.append('files', file._raw, file.name)) - let url = `${BASE}/upload` + // Use workspace-scoped endpoint when workspaceId is available. + // This writes to workspace_files table + saves to disk, fixing the + // "files disappear" bug where the legacy /upload wrote to uploaded_files + // but the agent context read from workspace_files. + let url const params = new URLSearchParams() if (sessionId) params.append('session_id', sessionId) - if (workspaceId) params.append('workspace_id', workspaceId) + + if (workspaceId) { + url = `${BASE}/workspaces/${workspaceId}/upload` + } else { + url = `${BASE}/upload` + } if (params.toString()) url += '?' + params.toString() return new Promise((resolve, reject) => { @@ -92,14 +101,14 @@ export async function uploadFiles(files, onProgress, sessionId = '', accessToken onProgress(file.id, rejected ? -1 : 100) }) resolve(data) - } catch (err) { + } catch { reject(new Error('Invalid JSON response')) } } else { try { const errData = JSON.parse(xhr.responseText) reject(new Error(errData.detail || errData.message || `Upload failed with status ${xhr.status}`)) - } catch (e) { + } catch { reject(new Error(`Upload failed with status ${xhr.status}`)) } } diff --git a/src/services/evalApi.js b/src/services/evalApi.js deleted file mode 100644 index 27169dd..0000000 --- a/src/services/evalApi.js +++ /dev/null @@ -1,81 +0,0 @@ -/** - * evalApi.js — Eval Dashboard API client - * - * All requests proxy through the Vite dev server to /api/eval/* - * Auth interceptor: every call accepts an optional accessToken and - * stamps it as Authorization: Bearer . - */ - -const BASE = '/api/eval' - -// --------------------------------------------------------------------------- -// Internal helpers -// --------------------------------------------------------------------------- - -/** - * @param {string|null|undefined} token - * @returns {Record} - */ -function authHeader(token) { - return token ? { Authorization: `Bearer ${token}` } : {} -} - -/** - * @param {string} path - * @param {string|null} [accessToken=null] - * @returns {Promise} - */ -async function _get(path, accessToken = null) { - const res = await fetch(`${BASE}${path}`, { - headers: { ...authHeader(accessToken) }, - }) - if (!res.ok) { - const err = await res.json().catch(() => ({ detail: `HTTP ${res.status}` })) - throw new Error(err.detail || `Request failed: ${res.status}`) - } - return res.json() -} - -// --------------------------------------------------------------------------- -// Eval endpoints (eval dashboard is intentionally public — no auth guard) -// --------------------------------------------------------------------------- - -/** System-level KPIs: total_runs, success_rate, avg_latency_ms, avg_retries */ -export async function getOverview(accessToken = null) { - return _get('/overview', accessToken) -} - -/** Per-agent stats: avg_latency_ms, failure_rate, total_calls */ -export async function getAgentPerformance(accessToken = null) { - return _get('/agents', accessToken) -} - -/** Debug loop stats: avg_debug_depth, error_type_distribution, retry_success_ratio */ -export async function getDebugLoopStats(accessToken = null) { - return _get('/debug-loop', accessToken) -} - -/** - * Paginated run list with metrics. - * - * @param {{ limit?: number, difficulty?: string, mode?: string }} [params={}] - * @param {string|null} [accessToken=null] - */ -export async function listEvalRuns(params = {}, accessToken = null) { - const qs = new URLSearchParams() - if (params.limit) qs.set('limit', params.limit) - if (params.difficulty) qs.set('difficulty', params.difficulty) - if (params.mode) qs.set('mode', params.mode) - const query = qs.toString() ? `?${qs}` : '' - return _get(`/runs${query}`, accessToken) -} - -/** - * Step-by-step trace for one run. - * - * @param {string} runId - * @param {string|null} [accessToken=null] - */ -export async function getRunTrace(runId, accessToken = null) { - return _get(`/runs/${runId}/trace`, accessToken) -} diff --git a/src/stores/workspaceStore.js b/src/stores/workspaceStore.js index ce7b446..a272681 100644 --- a/src/stores/workspaceStore.js +++ b/src/stores/workspaceStore.js @@ -5,6 +5,11 @@ * - The list of user workspaces fetched from Supabase. * - The currently active workspace (persisted to sessionStorage). * - CRUD actions: fetch, create, select. + * - Multi-file context: the MultiFileContext returned by the backend + * after a workspace upload, including FileSchema objects and inferred + * join candidates. + * - Join overrides: manually specified join key pairs that override the + * inferred candidates in SchemaPreviewPanel. * * The store is auth-aware: fetchWorkspaces() is a no-op when no * access token is provided, and createWorkspace() guards the same way. @@ -45,9 +50,34 @@ function persistWorkspaceId(id) { /** * @typedef {import('../types/index').Workspace} Workspace + * + * @typedef {Object} JoinCandidate + * @property {string} left_var + * @property {string} right_var + * @property {string} left_col + * @property {string} right_col + * @property {number} confidence + * @property {string} reason + * + * @typedef {Object} ColumnMeta + * @property {string} name + * @property {string} dtype + * @property {string[]} sample_values + * + * @typedef {Object} FileSchema + * @property {string} var_name + * @property {string} file_name + * @property {string} file_path + * @property {string} file_type + * @property {number} row_count + * @property {ColumnMeta[]} columns + * + * @typedef {Object} MultiFileContext + * @property {FileSchema[]} files + * @property {JoinCandidate[]} join_candidates */ -export const useWorkspaceStore = create((set, get) => ({ +export const useWorkspaceStore = create((set) => ({ /** @type {Workspace[]} */ workspaces: [], @@ -59,6 +89,24 @@ export const useWorkspaceStore = create((set, get) => ({ /** @type {string|null} */ error: null, + // ── Multi-file state ─────────────────────────────────────────────────────── + + /** + * The MultiFileContext returned by the backend after a workspace upload. + * Contains per-file schemas and inferred join candidates. + * + * @type {MultiFileContext|null} + */ + multiFileContext: null, + + /** + * User-specified join key overrides. Each entry replaces (not merges with) + * the inferred candidate at the same position when sent to the agent. + * + * @type {JoinCandidate[]} + */ + joinOverrides: [], + // ── Actions ────────────────────────────────────────────────────────────── /** @@ -124,12 +172,64 @@ export const useWorkspaceStore = create((set, get) => ({ /** * Sets the active workspace and persists the choice to sessionStorage. + * Clears the multi-file context because the new workspace has its own files. * * @param {Workspace} workspace */ setActiveWorkspace: (workspace) => { persistWorkspaceId(workspace?.id ?? null) - set({ activeWorkspace: workspace }) + set({ activeWorkspace: workspace, multiFileContext: null, joinOverrides: [] }) + }, + + /** + * Stores the MultiFileContext returned after a workspace file upload. + * Called by FileUploadPanel after each successful upload response. + * + * @param {MultiFileContext|null} ctx + */ + setMultiFileContext: (ctx) => { + set({ multiFileContext: ctx }) + }, + + /** + * Appends a manual join key override. + * The override must be a JoinCandidate-shaped object. + * Duplicates (same left_var+left_col+right_var+right_col) are replaced. + * + * @param {JoinCandidate} candidate + */ + addJoinOverride: (candidate) => { + set((state) => { + const existing = state.joinOverrides.filter( + (o) => + !( + o.left_var === candidate.left_var && + o.left_col === candidate.left_col && + o.right_var === candidate.right_var && + o.right_col === candidate.right_col + ) + ) + return { joinOverrides: [...existing, { ...candidate, confidence: 1.0, reason: 'manual override' }] } + }) + }, + + /** + * Removes the join override at the given index. + * + * @param {number} index + */ + removeJoinOverride: (index) => { + set((state) => ({ + joinOverrides: state.joinOverrides.filter((_, i) => i !== index), + })) + }, + + /** + * Clears the multi-file context and all overrides. + * Called when the user removes all files from the workspace. + */ + resetFileContext: () => { + set({ multiFileContext: null, joinOverrides: [] }) }, /** @@ -137,6 +237,13 @@ export const useWorkspaceStore = create((set, get) => ({ */ reset: () => { persistWorkspaceId(null) - set({ workspaces: [], activeWorkspace: null, loading: false, error: null }) + set({ + workspaces: [], + activeWorkspace: null, + loading: false, + error: null, + multiFileContext: null, + joinOverrides: [], + }) }, }))