Integrate the analytics

growth-agent/.env.example

@@ -2,10 +2,6 @@
 # Copy to .env and fill in values for local debugging and runtime-related values.
 # Deploy context comes from your active Scaleway profile in ~/.config/scw/config.yaml.
 
-# Umami Cloud API
-UMAMI_API_KEY=your-umami-api-key
-UMAMI_WEBSITE_ID=e41ae7d9-a536-426d-b40e-f2488b11bf95
-
 # LLM provider — "ionos" (default) or "mistral"
 LLM_PROVIDER=ionos
 # Optional: override the provider's default model (leave blank to use default)
@@ -17,7 +13,8 @@ IONOS_API_TOKEN=your-ionos-api-token
 # Mistral API (used when LLM_PROVIDER=mistral)
 MISTRAL_API_KEY=your-mistral-api-key
 
-# Scaleway S3
+# Scaleway S3 — also powers _collect_page_traffic's read of the analytics
+# service's rollups (rollup/{site}/{month}.json at the bucket root)
 SCW_ACCESS_KEY=your-scw-access-key
 SCW_SECRET_KEY=your-scw-secret-key
 S3_BUCKET=my-imagestore

growth-agent/GROWTH_AGENT.md

@@ -2,6 +2,13 @@
 
 ## Implementation Plan & Architecture
 
+> **Note:** this plan predates the migration off Umami Cloud. Umami has been
+> fully replaced by the self-hosted `analytics` service (see
+> `../analytics/README.md`); every reference to Umami below is historical.
+> Page traffic is now collected via `_collect_page_traffic` in
+> `agent/nodes/ingest.py`, reading the `analytics` service's own S3 rollups —
+> `agent/umami_client.py` no longer exists.
+
 ---
 
 # 1. Objective
@@ -177,7 +184,6 @@ growth-agent/
 │   ├── __init__.py
 │   ├── models.py           # Pydantic state models (Insights, Strategy, Draft, LLMAnalysis, PageMeta, ...)
 │   ├── llm_client.py       # IONOS LLM client (langchain-openai ChatOpenAI)
-│   ├── umami_client.py     # Umami Cloud REST API client
 │   ├── page_meta.py        # HTTP-based page metadata fetcher (title, description from meta tags)
 │   ├── storage.py          # LocalStorage (notebooks) + S3Storage (production) + load_model helper
 │   ├── publisher.py        # Publish approved drafts to platforms
@@ -186,7 +192,7 @@ growth-agent/
 │   │   └── bluesky.py      # AT Protocol client (httpx)
 │   ├── nodes/               # LangGraph node modules
 │   │   ├── __init__.py
-│   │   ├── ingest.py       # ingest_analytics() — Umami + social metrics
+│   │   ├── ingest.py       # ingest_analytics() — page traffic + social metrics
 │   │   ├── insights.py     # generate_insights() — LLM analysis + prompt templates
 │   │   ├── strategy.py     # adjust_strategy() — LLM strategy adjustments + audit log
 │   │   ├── plan.py         # create_plan() — page selection, scheduling, pipeline depth
@@ -196,7 +202,7 @@ growth-agent/
 │   └── graph.py            # LangGraph StateGraph — OODA loop compilation
 │
 ├── notebooks/
-│   ├── 01_umami_ingest.ipynb
+│   ├── 01_analytics_ingest.ipynb
 │   ├── 02_llm_insights.ipynb
 │   ├── 03_content_creation.ipynb
 │   ├── 04_s3_state.ipynb

growth-agent/agent/models.py

@@ -151,6 +151,9 @@ class Performance(BaseModel):
     """Aggregated engagement metrics for all published posts."""
 
     posts: list[PostMetrics] = Field(default_factory=list)
+    # Page path -> hits in the trailing 30 days, from the analytics service's
+    # monthly rollups. See agent/nodes/ingest.py's _collect_page_traffic.
+    page_traffic: dict[str, int] = Field(default_factory=dict)
 
 
 class DraftCritique(BaseModel):

growth-agent/agent/nodes/ingest.py

@@ -2,7 +2,8 @@
 
 import logging
 import os
-from datetime import datetime, timedelta, timezone
+from collections import defaultdict
+from datetime import date, datetime, timedelta, timezone
 
 from agent.models import (
     ContentQueue,
@@ -14,7 +15,7 @@
 from agent.platforms.bluesky import BlueskyClient
 from agent.platforms.mastodon import MastodonClient
 from agent.state import AgentState
-from agent.storage import load_model
+from agent.storage import S3Storage, load_model
 
 logger = logging.getLogger("growth-agent")
 
@@ -65,6 +66,12 @@ def ingest_analytics(storage) -> Insights:
     # Per-post engagement metrics
     _collect_post_metrics(storage)
 
+    # Page traffic (last 30 days). _collect_post_metrics's write now carries its
+    # own page_traffic forward, so this ordering is defense-in-depth rather than
+    # load-bearing — but keeping it after post metrics still means this always
+    # merges onto the freshest performance.json.
+    _collect_page_traffic(storage)
+
     return insights
 
 
@@ -155,7 +162,76 @@ def _collect_post_metrics(storage) -> None:
                 logger.exception("Bluesky per-post metrics failed")
 
         refreshed = len(recent_mastodon) + len(recent_bluesky)
-        storage.write("performance.json", Performance(posts=list(updated.values())))
+        storage.write(
+            "performance.json",
+            Performance(posts=list(updated.values()), page_traffic=existing.page_traffic),
+        )
         logger.info("Per-post metrics: %d total, %d refreshed", len(updated), refreshed)
     except Exception:
         logger.exception("Per-post metrics collection failed")
+
+
+PAGE_TRAFFIC_WINDOW_DAYS = 30  # matches _METRICS_REFRESH_DAYS's existing precedent
+ANALYTICS_SITE = "fretchen.eu"  # matches analytics/hit.ts's own hardcoded SITE
+
+
+def _months_between(start: date, end: date) -> list[str]:
+    """Every YYYY-MM the [start, end] range touches, inclusive."""
+    months = []
+    y, m = start.year, start.month
+    while (y, m) <= (end.year, end.month):
+        months.append(f"{y:04d}-{m:02d}")
+        m += 1
+        if m > 12:
+            m, y = 1, y + 1
+    return months
+
+
+def _sum_trailing_hits(rollups: dict[str, dict], window_start: date, today: date) -> dict[str, int]:
+    """Sum each page's hits for days in [window_start, today] across already-fetched
+    rollup objects (keyed YYYY-MM). Split out from the S3/env-var plumbing in
+    _collect_page_traffic so the month-boundary logic is testable without mocking S3.
+    """
+    hits_by_page: dict[str, int] = defaultdict(int)
+    for rollup in rollups.values():
+        for day, bucket in rollup.get("days", {}).items():
+            if window_start.isoformat() <= day <= today.isoformat():
+                for path, hits in bucket.get("pages", {}).items():
+                    hits_by_page[path] += hits
+    return dict(hits_by_page)
+
+
+def _collect_page_traffic(storage) -> None:
+    """Trailing-30-day per-page hits from the analytics service's monthly rollups,
+    merged into performance.json.
+
+    Reads from the same S3 bucket growth-agent already holds credentials for
+    (`rollup/{site}/{YYYY-MM}.json`, written by the separate `analytics` service) —
+    a second S3Storage instance with an empty prefix, not a new secret or HTTP
+    client. Rollup-only: recent days the weekly compaction cron hasn't reached yet
+    are simply absent from the sum, an accepted, self-healing staleness gap of up
+    to ~6 days rather than replicating the hourly-bucket fallback here too.
+    """
+    try:
+        analytics_storage = S3Storage(
+            bucket=os.environ["S3_BUCKET"],
+            prefix="",
+            access_key=os.environ["SCW_ACCESS_KEY"],
+            secret_key=os.environ["SCW_SECRET_KEY"],
+        )
+        today = datetime.now(timezone.utc).date()
+        window_start = today - timedelta(days=PAGE_TRAFFIC_WINDOW_DAYS - 1)
+        months = _months_between(window_start, today)
+        rollups: dict[str, dict] = {}
+        for month in months:
+            data = analytics_storage.read(f"rollup/{ANALYTICS_SITE}/{month}.json")
+            rollups[month] = data if isinstance(data, dict) else {"days": {}}
+
+        hits_by_page = _sum_trailing_hits(rollups, window_start, today)
+
+        performance = load_model(storage, "performance.json", Performance)
+        performance.page_traffic = hits_by_page
+        storage.write("performance.json", performance)
+        logger.info("Page traffic: %d pages, window %s..%s", len(hits_by_page), window_start, today)
+    except Exception:
+        logger.exception("Page traffic collection failed")