From 96c88fb84e4b7cf868cb9b8e64e60ac46a150ea3 Mon Sep 17 00:00:00 2001 From: Hunter Hillman Date: Mon, 20 Apr 2026 15:38:47 -0700 Subject: [PATCH 01/11] feat: agentic workflow builder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an in-app agent that translates natural-language intent into working Scope graphs and runtime parameter tweaks. Accessible from a new Agent button next to Graph in the toolbar; a right-side resizable drawer hosts the conversation, streaming responses, and workflow-proposal cards. Structural changes (new graph / pipeline load) require an explicit approve step; runtime params (prompts, noise, LoRA weights) auto-apply. Multi-provider BYOK: Anthropic (default), any OpenAI-compatible endpoint (OpenAI, OpenRouter, Groq, together.ai, Fireworks), and self-hosted (Ollama, vLLM, LM Studio). Provider + model are configured under a new Settings → Agent tab; API keys continue to live in the API Keys tab (extended to cover anthropic, openai, llm_custom). Agent tools are pure-Python in `agent_tool_impls.py` so both the MCP server and the in-app agent share the same surface. Everything the agent knows about Scope is discovered at runtime through introspection tools (pipeline registry, schema metadata, blueprints, LoRAs, assets, node-type manifest), so new pipelines and nodes work on day 0 without touching agent code. Backend - `src/scope/server/agent_tool_impls.py`: shared tool surface - `src/scope/server/agent_state.py`: session store + provider config - `src/scope/server/agent_providers.py`: Anthropic + OpenAI-compat + self-hosted providers behind a single event protocol - `src/scope/server/agent_loop.py`: SSE turn runner with propose→approve handshake and vision feedback - `src/scope/server/app.py`: `/api/v1/agent/chat` (SSE), `/agent/decision`, `/agent/config`, `/agent/sessions`, `/agent/node-catalog`; `/api/v1/keys/*` extended - `anthropic>=0.40` added to pyproject Frontend - `frontend/src/components/agent/`: AgentDrawer, ChatTranscript, MessageBubble, ToolCallBlock, Composer, WorkflowProposalCard - `frontend/src/contexts/AgentContext.tsx` + `lib/agentClient.ts`: state + SSE-over-POST reader - `frontend/src/components/settings/AgentProviderTab.tsx`: new settings tab, wired into SettingsDialog + Header - `frontend/src/components/graph/GraphToolbar.tsx`: Agent button - `frontend/src/data/nodes/manifest.json`: UI node-type catalog so the agent can compose arbitrary node graphs without hardcoding Signed-off-by: Hunter Hillman --- .claude/launch.json | 9 + frontend/src/App.tsx | 7 +- frontend/src/components/Header.tsx | 8 +- frontend/src/components/SettingsDialog.tsx | 11 + frontend/src/components/agent/AgentDrawer.tsx | 173 ++++ .../src/components/agent/ChatTranscript.tsx | 73 ++ frontend/src/components/agent/Composer.tsx | 54 ++ .../src/components/agent/MessageBubble.tsx | 40 + .../src/components/agent/ToolCallBlock.tsx | 56 ++ .../components/agent/WorkflowProposalCard.tsx | 135 +++ .../src/components/graph/GraphToolbar.tsx | 21 + .../components/settings/AgentProviderTab.tsx | 203 +++++ frontend/src/contexts/AgentContext.tsx | 365 ++++++++ frontend/src/data/nodes/manifest.json | 261 ++++++ frontend/src/lib/agentClient.ts | 100 +++ frontend/src/lib/api.ts | 134 +++ pyproject.toml | 1 + src/scope/server/agent_loop.py | 428 +++++++++ src/scope/server/agent_providers.py | 532 +++++++++++ src/scope/server/agent_state.py | 294 ++++++ src/scope/server/agent_tool_impls.py | 834 ++++++++++++++++++ src/scope/server/app.py | 305 ++++++- uv.lock | 120 +++ 23 files changed, 4160 insertions(+), 4 deletions(-) create mode 100644 frontend/src/components/agent/AgentDrawer.tsx create mode 100644 frontend/src/components/agent/ChatTranscript.tsx create mode 100644 frontend/src/components/agent/Composer.tsx create mode 100644 frontend/src/components/agent/MessageBubble.tsx create mode 100644 frontend/src/components/agent/ToolCallBlock.tsx create mode 100644 frontend/src/components/agent/WorkflowProposalCard.tsx create mode 100644 frontend/src/components/settings/AgentProviderTab.tsx create mode 100644 frontend/src/contexts/AgentContext.tsx create mode 100644 frontend/src/data/nodes/manifest.json create mode 100644 frontend/src/lib/agentClient.ts create mode 100644 src/scope/server/agent_loop.py create mode 100644 src/scope/server/agent_providers.py create mode 100644 src/scope/server/agent_state.py create mode 100644 src/scope/server/agent_tool_impls.py diff --git a/.claude/launch.json b/.claude/launch.json index 9ccce1cb4..17b61e275 100644 --- a/.claude/launch.json +++ b/.claude/launch.json @@ -6,6 +6,15 @@ "runtimeExecutable": "bash", "runtimeArgs": ["-c", "source ~/.nvm/nvm.sh && cd frontend && npm run dev"], "port": 5173 + }, + { + "name": "backend", + "runtimeExecutable": "bash", + "runtimeArgs": [ + "-c", + "CUDA_VISIBLE_DEVICES='' uv run daydream-scope --port 8033" + ], + "port": 8033 } ] } diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index cf289cc1d..27a00d623 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -9,6 +9,8 @@ import { CloudProvider } from "./lib/cloudContext"; import { CloudStatusProvider } from "./hooks/useCloudStatus"; import { OnboardingProvider } from "./contexts/OnboardingContext"; import { BillingProvider } from "./contexts/BillingContext"; +import { AgentProvider } from "./contexts/AgentContext"; +import { AgentDrawer } from "./components/agent/AgentDrawer"; import { handleOAuthCallback, initElectronAuthListener, @@ -115,7 +117,10 @@ function App() { - + + + + diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx index c3927d901..54f5bfc7d 100644 --- a/frontend/src/components/Header.tsx +++ b/frontend/src/components/Header.tsx @@ -70,7 +70,13 @@ export function Header({ const [settingsOpen, setSettingsOpen] = useState(false); const [pluginsOpen, setPluginsOpen] = useState(false); const [initialTab, setInitialTab] = useState< - "general" | "account" | "api-keys" | "loras" | "osc" | "billing" + | "general" + | "account" + | "api-keys" + | "agent" + | "loras" + | "osc" + | "billing" >("general"); const [initialPluginPath, setInitialPluginPath] = useState(""); const [pluginsInitialTab, setPluginsInitialTab] = useState< diff --git a/frontend/src/components/SettingsDialog.tsx b/frontend/src/components/SettingsDialog.tsx index 83cc6fcef..e0fed58be 100644 --- a/frontend/src/components/SettingsDialog.tsx +++ b/frontend/src/components/SettingsDialog.tsx @@ -2,6 +2,7 @@ import { useState, useEffect } from "react"; import { Dialog, DialogContent } from "./ui/dialog"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "./ui/tabs"; import { AccountTab } from "./settings/AccountTab"; +import { AgentProviderTab } from "./settings/AgentProviderTab"; import { ApiKeysTab } from "./settings/ApiKeysTab"; import { GeneralTab } from "./settings/GeneralTab"; import { ReportBugDialog } from "./ReportBugDialog"; @@ -24,6 +25,7 @@ interface SettingsDialogProps { | "account" | "billing" | "api-keys" + | "agent" | "loras" | "osc" | "dmx" @@ -160,6 +162,12 @@ export function SettingsDialog({ > API Keys + + Agent + + + + (() => { + const stored = localStorage.getItem(DRAWER_WIDTH_KEY); + const parsed = stored ? parseInt(stored, 10) : NaN; + return Number.isFinite(parsed) + ? Math.min(MAX_WIDTH, Math.max(MIN_WIDTH, parsed)) + : DEFAULT_WIDTH; + }); + const draggingRef = useRef(false); + + useEffect(() => { + localStorage.setItem(DRAWER_WIDTH_KEY, String(width)); + }, [width]); + + const onDragStart = useCallback( + (e: React.MouseEvent) => { + e.preventDefault(); + draggingRef.current = true; + const startX = e.clientX; + const startWidth = width; + const onMove = (me: MouseEvent) => { + if (!draggingRef.current) return; + const delta = startX - me.clientX; + const next = Math.min( + MAX_WIDTH, + Math.max(MIN_WIDTH, startWidth + delta) + ); + setWidth(next); + }; + const onUp = () => { + draggingRef.current = false; + window.removeEventListener("mousemove", onMove); + window.removeEventListener("mouseup", onUp); + }; + window.addEventListener("mousemove", onMove); + window.addEventListener("mouseup", onUp); + }, + [width] + ); + + if (!drawerOpen) return null; + + const needsKey = + !!config && + !configError && + config.key_sources[config.provider] == null && + config.provider !== "self_hosted"; + + return ( +
+ {/* Resize handle */} +
+ + {/* Header */} +
+
+
Scope Agent
+ {config && ( + + {config.provider === "anthropic" + ? `Claude • ${config.model}` + : config.provider === "openai_compatible" + ? `OpenAI • ${config.model}` + : `Local • ${config.model}`} + + )} +
+
+ {isStreaming && ( + + )} + + +
+
+ + {/* Banner: missing key / config error */} + {configError && ( +
+ Failed to load agent config: {configError} +
+ )} + {needsKey && ( +
+ No API key configured for{" "} + {config?.provider === "anthropic" ? "Anthropic" : "OpenAI-compatible"} + . Open Settings → API Keys to add one. +
+ )} + + {/* Transcript */} + + + {/* Composer */} + +
+ ); +} diff --git a/frontend/src/components/agent/ChatTranscript.tsx b/frontend/src/components/agent/ChatTranscript.tsx new file mode 100644 index 000000000..11976ec21 --- /dev/null +++ b/frontend/src/components/agent/ChatTranscript.tsx @@ -0,0 +1,73 @@ +import { useEffect, useRef } from "react"; +import type { AgentMessage, AgentProposal } from "@/contexts/AgentContext"; +import { MessageBubble } from "./MessageBubble"; +import { WorkflowProposalCard } from "./WorkflowProposalCard"; + +interface ChatTranscriptProps { + messages: AgentMessage[]; + pendingProposal: AgentProposal | null; + onDecide: (approved: boolean, reason?: string) => Promise; +} + +export function ChatTranscript({ + messages, + pendingProposal, + onDecide, +}: ChatTranscriptProps) { + const scrollRef = useRef(null); + const stickyBottomRef = useRef(true); + + // Track whether user is at the bottom. If so, auto-scroll; otherwise leave + // their scroll position alone. + const onScroll = (e: React.UIEvent) => { + const el = e.currentTarget; + stickyBottomRef.current = + el.scrollHeight - el.scrollTop - el.clientHeight < 48; + }; + + useEffect(() => { + if (!stickyBottomRef.current) return; + const el = scrollRef.current; + if (el) el.scrollTop = el.scrollHeight; + }, [messages, pendingProposal]); + + if (messages.length === 0 && !pendingProposal) { + return ( +
+

+ Tell me what you want to build. +

+

+ I can pick pipelines, compose workflows, and tune parameters by + watching the output. +

+
    +
  • + • "Hyperrealistic scene with 3–5 switchable prompts" +
    • +
  • + • "It's not recognizing depth well" +
    • +
  • + • "Help me record what I'm seeing" +
    • +
+
+ ); + } + + return ( +
+ {messages.map(m => ( + + ))} + {pendingProposal && !pendingProposal.decision && ( + + )} +
+ ); +} diff --git a/frontend/src/components/agent/Composer.tsx b/frontend/src/components/agent/Composer.tsx new file mode 100644 index 000000000..de275f934 --- /dev/null +++ b/frontend/src/components/agent/Composer.tsx @@ -0,0 +1,54 @@ +import { useRef, useState, type KeyboardEvent } from "react"; +import { Send } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +interface ComposerProps { + onSend: (text: string) => Promise; + disabled?: boolean; + placeholder?: string; +} + +export function Composer({ onSend, disabled, placeholder }: ComposerProps) { + const [value, setValue] = useState(""); + const textareaRef = useRef(null); + + const send = async () => { + const text = value.trim(); + if (!text || disabled) return; + setValue(""); + await onSend(text); + textareaRef.current?.focus(); + }; + + const onKeyDown = (e: KeyboardEvent) => { + // Cmd/Ctrl+Enter or bare Enter to send (Shift+Enter inserts newline). + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + void send(); + } + }; + + return ( +
+