Migrate to MCP, closes #29

.env.example

@@ -7,7 +7,7 @@ PRODUCTION=false
 # =====================================================================
 # Agent & AI Provider Settings
 # =====================================================================
-# Agent Provider Options: openai, julep
+# Agent Provider Options: openai
 AGENT_PROVIDER=openai
 
 # Microsoft Azure Foundry (OpenAI-compatible)
@@ -20,10 +20,6 @@ OPENAI_API_KEY=your-openai-api-key
 OPENAI_MODEL=gpt-4o-mini
 OPENAI_BASE_URL=
 
-# Julep Settings (when AGENT_PROVIDER=julep or RAG_BACKEND=julep)
-# JULEP_API_KEY=your-julep-api-key
-# JULEP_MODEL=claude-3.5-sonnet
-
 # =====================================================================
 # RAG & Embedding Settings
 # =====================================================================

Dockerfile.mcp

@@ -0,0 +1,14 @@
+FROM python:3.13-slim
+
+WORKDIR /app
+
+COPY mcp_server/requirements.txt mcp_server/requirements.txt
+RUN pip install --no-cache-dir -r mcp_server/requirements.txt
+
+COPY config.py .
+COPY service/ service/
+COPY mcp_server/ mcp_server/
+
+EXPOSE 8002
+
+CMD ["python", "-m", "mcp_server"]

config.py

@@ -72,22 +72,6 @@ def OPENAI_EMBEDDING_MODEL(self) -> str:
             or os.getenv("OPENAI_EMBEDDING_MODEL", "text-embedding-3-small")
         )
 
-    # Julep Provider
-    @property
-    def JULEP_API_KEY(self) -> str | None:
-        return os.getenv("JULEP_API_KEY")
-
-    @property
-    def JULEP_MODEL(self) -> str:
-        return (
-            os.getenv("AGENT_MODEL")
-            or os.getenv("JULEP_MODEL", "claude-3.5-sonnet")
-        )
-
-    @property
-    def JULEP_ENVIRONMENT(self) -> str:
-        return os.getenv("JULEP_ENVIRONMENT", "production")
-
     # ------------------------------------------------------------------
     # Local Sentence Transformers Settings
     # ------------------------------------------------------------------

dev.sh

@@ -57,6 +57,7 @@ if [ "$DO_INSTALL" = true ]; then
     $VENV_PYTHON -m pip install -r server/requirements.txt \
                                -r embedding_server/requirements.txt \
                                -r pipeline/requirements.txt \
+                               -r mcp_server/requirements.txt \
                                pytest pytest-asyncio email-validator azure-storage-blob
     if [ -d "frontend" ] && [ -f "frontend/package.json" ]; then
         echo -e "${CYAN}📦 Installing frontend npm dependencies...${NC}"
@@ -81,7 +82,7 @@ PIDS=()
 
 cleanup() {
     trap - INT TERM EXIT
-    echo -e "\n${CYAN}🛑 Stopping local servers...${NC}"
+    echo -e "\n${CYAN}D Stopping local servers...${NC}"
     for pid in "${PIDS[@]}"; do
         if kill -0 "$pid" 2>/dev/null; then
             kill -TERM "$pid" 2>/dev/null || true
@@ -104,6 +105,13 @@ echo -e "${CYAN}🧠 Starting Embedding Server on http://localhost:8001...${NC}"
 $VENV_PYTHON -m uvicorn embedding_server.app:app --host 0.0.0.0 --port 8001 &
 PIDS+=($!)
 
+# Start MCP Server (SSE) in background
+if $VENV_PYTHON -c "import mcp" 2>/dev/null; then
+    echo -e "${CYAN}🔌 Starting MCP Server (SSE) on http://localhost:8002...${NC}"
+    $VENV_PYTHON -c "from mcp_server.app import mcp; mcp.run(transport='sse', port=8002)" &
+    PIDS+=($!)
+fi
+
 # Start Web Backend Server with Hot-Reloading in background
 echo -e "${CYAN}🌐 Starting Web Backend Server (hot-reloading) on http://localhost:8000...${NC}"
 $VENV_PYTHON -m uvicorn server.app:app --reload --host 0.0.0.0 --port 8000 &

docker-compose.yml

@@ -43,6 +43,27 @@ services:
     networks:
       - default
 
+  mcp-server:
+    build:
+      context: .
+      dockerfile: Dockerfile.mcp
+    ports:
+      - "8002:8002"
+    environment:
+      EMBEDDING_SERVICE_URL: http://embedding-server:8001
+      DB_URL: ${DB_URL:-mongodb://mongo:27017/evolution}
+      ARTICLE_STORE_BACKEND: file
+      RAG_BACKEND: memory
+    env_file:
+      - .env
+    volumes:
+      - ./data:/app/data
+    depends_on:
+      - embedding-server
+    restart: unless-stopped
+    networks:
+      - default
+
   backend:
     build:
       context: .

mcp_server/__init__.py

@@ -0,0 +1 @@
+"""DistillNews MCP Server package."""

mcp_server/__main__.py

@@ -0,0 +1,2 @@
+from mcp_server.app import main
+main()

mcp_server/app.py

@@ -0,0 +1,32 @@
+from mcp.server.fastmcp import FastMCP
+
+mcp = FastMCP("DistillNews Engine")
+
+@mcp.tool()
+def news_search(query: str, limit: int = 5, category: str | None = None) -> list[dict]:
+    """Search the DistillNews corpus for relevant news articles using vector similarity and keyword matching."""
+    from mcp_server.tools.search import search_news
+    return search_news(query=query, limit=limit, category=category)
+
+@mcp.tool()
+def get_article(article_id: str) -> dict:
+    """Retrieve the full content and metadata of a specific article by its ID."""
+    from mcp_server.tools.articles import fetch_article
+    return fetch_article(article_id=article_id)
+
+@mcp.tool()
+def list_categories() -> list[str]:
+    """List all available news categories in the corpus."""
+    return ["World", "Business", "Technology", "Entertainment", "Sports", "Science", "Health"]
+
+@mcp.tool()
+def get_article_count() -> dict:
+    """Get the total number of articles in the corpus."""
+    from mcp_server.tools.articles import count_articles
+    return count_articles()
+
+def main():
+    mcp.run(transport="stdio")
+
+if __name__ == "__main__":
+    main()

mcp_server/requirements.txt

@@ -0,0 +1,5 @@
+mcp>=1.0.0
+fastmcp
+pymongo
+requests
+python-dotenv

mcp_server/tools/articles.py

@@ -0,0 +1,30 @@
+from service.db import create_article_store
+
+_article_store = None
+
+def _get_store():
+    global _article_store
+    if _article_store is None:
+        _article_store = create_article_store()
+    return _article_store
+
+def fetch_article(article_id: str) -> dict:
+    store = _get_store()
+    article = store.load_article(article_id)
+    if not article:
+        return {"error": f"Article with ID {article_id} not found."}
+
+    return {
+        "title": article.get("title", ""),
+        "content": article.get("content", article.get("markdown_content", "")),
+        "category": article.get("category", ""),
+        "tags": article.get("tags", []),
+        "summary": article.get("summary", ""),
+        "publication_date": article.get("publication_date", "")
+    }
+
+def count_articles() -> dict:
+    store = _get_store()
+    # list_articles returns lightweight metadata for stored articles
+    articles = store.list_articles()
+    return {"total": len(articles)}

mcp_server/tools/search.py

@@ -0,0 +1,59 @@
+from service.rag.providers.remote_embedding import RemoteEmbeddingProvider
+from service.rag.backends.memory import InMemoryVectorStore
+from service.rag.base import Document
+from service.db import create_article_store
+
+_store = None
+_loaded = False
+
+def _ensure_store():
+    global _store, _loaded
+    if _loaded:
+        return
+
+    embedder = RemoteEmbeddingProvider()
+    _store = InMemoryVectorStore(embedder=embedder)
+
+    article_store = create_article_store()
+    articles = article_store.load_all_articles()
+
+    documents = []
+    for art in articles:
+        content = art.get("content", art.get("markdown_content", ""))
+        metadata = {
+            "id": art.get("id", ""),
+            "category": art.get("category", ""),
+            "tags": art.get("tags", []),
+            "summary": art.get("summary", ""),
+            "publication_date": art.get("publication_date", "")
+        }
+        documents.append(Document(
+            title=art.get("title", ""),
+            content=content,
+            metadata=metadata
+        ))
+
+    _store.upload(documents)
+    _loaded = True
+
+def search_news(query: str, limit: int = 5, category: str | None = None) -> list[dict]:
+    _ensure_store()
+    results = _store.search(query=query, limit=limit if not category else limit * 5)
+
+    formatted_results = []
+    for res in results:
+        if category and res.metadata.get("category", "").lower() != category.lower():
+            continue
+
+        formatted_results.append({
+            "id": res.metadata.get("id", ""),
+            "title": res.title,
+            "snippet": res.snippet,
+            "category": res.metadata.get("category", ""),
+            "score": res.score
+        })
+
+        if len(formatted_results) >= limit:
+            break
+
+    return formatted_results

pipeline/extraction_schemas.py

@@ -0,0 +1,65 @@
+"""JSON Schema definitions for structured extraction via LLM tool calls.
+
+These schemas replace raw text/JSON parsing with type-safe function call schemas,
+ensuring 100% valid structured output from the extraction pipeline.
+"""
+
+from service.agents.base import ToolDefinition
+
+
+ARTICLE_EXTRACTION_TOOL = ToolDefinition(
+    name="submit_extracted_article",
+    description="Submit the extracted and structured news article metadata. Call this exactly once with all extracted fields.",
+    parameters={
+        "type": "object",
+        "properties": {
+            "title": {"type": "string", "description": "The article headline."},
+            "publication_date": {"type": "string", "description": "Publication date in ISO 8601 format."},
+            "summary": {"type": "string", "description": "A concise summary of the article, maximum 100 words."},
+            "content": {
+                "type": "string",
+                "description": "The full article body text. Preserve paragraph breaks. Write in third-person voice for community sources.",
+            },
+            "category": {
+                "type": "string",
+                "enum": ["World", "Business", "Technology", "Entertainment", "Sports", "Science", "Health"],
+                "description": "The primary news category.",
+            },
+            "tags": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Relevant keyword tags (e.g., politics, economy, AI).",
+            },
+            "location": {
+                "type": "string",
+                "description": "Geographic location mentioned or inferred from the article. Use 'unknown' if not inferable.",
+            },
+        },
+        "required": ["title", "publication_date", "summary", "content", "category", "tags", "location"],
+    },
+)
+
+NEWS_CLASSIFICATION_TOOL = ToolDefinition(
+    name="submit_classification",
+    description="Submit whether this content is a newsworthy article or not.",
+    parameters={
+        "type": "object",
+        "properties": {
+            "is_news": {"type": "boolean", "description": "True if the content is newsworthy, false otherwise."},
+            "reason": {"type": "string", "description": "Brief explanation of the classification decision."},
+        },
+        "required": ["is_news"],
+    },
+)
+
+MARKDOWN_FORMAT_TOOL = ToolDefinition(
+    name="submit_formatted_content",
+    description="Submit the markdown-formatted version of the article content.",
+    parameters={
+        "type": "object",
+        "properties": {
+            "markdown": {"type": "string", "description": "The article content formatted with markdown headings, bullet points, and block quotes."},
+        },
+        "required": ["markdown"],
+    },
+)

service/agents/__init__.py

@@ -10,6 +10,14 @@
 """
 
 from .factory import create_agent
-from .base import AgentProvider, CompletionResult
+from .base import AgentProvider, CompletionResult, ToolCallingProvider, ToolDefinition, ToolCall, AgentMessage
 
-__all__ = ["create_agent", "AgentProvider", "CompletionResult"]
+__all__ = [
+    "create_agent",
+    "AgentProvider",
+    "CompletionResult",
+    "ToolCallingProvider",
+    "ToolDefinition",
+    "ToolCall",
+    "AgentMessage",
+]

service/agents/base.py

@@ -17,6 +17,41 @@ class CompletionResult:
     raw: dict | None = field(default=None, repr=False)  # Provider-specific raw response
 
 
+@dataclass
+class ToolDefinition:
+    """Schema definition for a tool the LLM can invoke."""
+    name: str
+    description: str
+    parameters: dict  # JSON Schema
+
+    def to_openai_schema(self) -> dict:
+        return {
+            "type": "function",
+            "function": {
+                "name": self.name,
+                "description": self.description,
+                "parameters": self.parameters,
+            },
+        }
+
+
+@dataclass
+class ToolCall:
+    """A tool invocation returned by the LLM."""
+    id: str
+    name: str
+    arguments: dict
+
+
+@dataclass
+class AgentMessage:
+    """A single message in a multi-turn tool-calling conversation."""
+    role: str  # system | user | assistant | tool
+    content: str | None = None
+    tool_calls: list[ToolCall] | None = None
+    tool_call_id: str | None = None
+
+
 class AgentProvider(ABC):
     """Abstract base for LLM completion backends.
 
@@ -100,3 +135,16 @@ def _replacer(match: re.Match) -> str:
             return re.sub(r"\{steps\[0\]\.input\.(\w+)\}", _replacer, text)
 
         return _substitute(system_content), _substitute(user_content)
+
+
+class ToolCallingProvider(AgentProvider):
+    """Extended provider supporting native tool / function calling."""
+
+    @abstractmethod
+    def chat_with_tools(
+        self,
+        messages: list[AgentMessage],
+        tools: list[ToolDefinition] | None = None,
+        tool_choice: str | dict = "auto",
+    ) -> AgentMessage:
+        """Execute a chat completion with optional tool definitions."""