Add Antfly-native ingestion control plane

.env.example

@@ -8,6 +8,14 @@ ANTFLY_INFERENCE_URL=https://platform.antfly.io/cloud/v1/INSTANCE_ID/ai/v1
 # Paste only the token value; do not include "Bearer ".
 ANTFLY_API_KEY=
 
+# Antfly-native connector and durable ingestion-job API. Keep this write-capable
+# credential separate from the read-only support-agent key above.
+# Set this when the Antfly knowledge-ingestion API is enabled for your instance.
+ANTFLY_INGESTION_URL=
+ANTFLY_INGESTION_API_KEY=
+# Protects this template's /api/admin/ingestion routes. Use a long random value.
+KNOWLEDGE_ADMIN_TOKEN=
+
 # Antfly document table and indexes
 ANTFLY_TABLE=docs
 ANTFLY_VECTOR_INDEX=document_vectors

README.md

@@ -14,14 +14,22 @@ documentation search experience, and grounded support agent with:
 - Reusable `⌘K` / `Ctrl+K` support command palette with an embedded chat modal
 - A server-only deployment-readiness dashboard at `/admin`
 
-The current foundation provides the production support-agent path. Website,
-GitHub, and file connectors; conventional search; durable conversations;
-analytics; content-gap detection; and escalation adapters are the next staged
-modules. Their product boundary and release gates are maintained in the
-`knowledge-support` entry in the Antfly template library.
+The current foundation provides the production support-agent path and an
+Antfly-native ingestion control plane at `/admin/sources`. Sitemap, GitHub, S3,
+and file connectors execute as durable Antfly backend jobs. Conventional
+search, durable conversations, analytics, content-gap detection, and escalation
+adapters are the next staged modules. Their product boundary and release gates
+are maintained in the `knowledge-support` entry in the Antfly template library.
 
 The browser never receives Antfly or model-provider credentials. Retrieval and generation run in server-side Next.js routes.
 
+The template does not crawl, parse, or queue ingestion work in Vercel. See the
+[Antfly ingestion contract](docs/ANTFLY_INGESTION_CONTRACT.md) and
+[connector implementation brief](docs/ANTFLY_CONNECTOR_IMPLEMENTATION.md).
+
+For the component-level product gap and recommended 80% milestone, see the
+[Kapa React comparison](docs/KAPA_REACT_COMPARISON.md).
+
 ## Support command palette
 
 The reference page mounts `SupportCommandPalette` once alongside the full-page

app/admin/page.tsx

@@ -6,7 +6,7 @@ const surfaces = [
   {
     title: "Knowledge sources",
     description: "Website, sitemap, GitHub, and file synchronization.",
-    status: "Foundation",
+    status: "Antfly jobs",
   },
   {
     title: "Search and answers",

app/admin/sources/page.tsx

@@ -0,0 +1,6 @@
+import Link from "next/link";
+import { KnowledgeSources } from "@/components/knowledge-sources";
+
+export default function SourcesPage() {
+  return <main className="admin-shell"><header className="admin-header"><div><span className="admin-eyebrow">Knowledge operations</span><h1>Sources</h1><p>Configure knowledge sources and monitor the Antfly jobs that ingest them.</p></div><Link className="admin-primary-link" href="/admin">Admin home</Link></header><KnowledgeSources /></main>;
+}

app/api/admin/ingestion/jobs/[jobId]/cancel/route.ts

@@ -0,0 +1,13 @@
+import { isAdminRequest } from "@/lib/ingestion/auth";
+import { antflyIngestion, AntflyIngestionError } from "@/lib/ingestion/client";
+
+export async function POST(request: Request, context: { params: Promise<{ jobId: string }> }) {
+  if (!isAdminRequest(request)) return Response.json({ error: "Unauthorized" }, { status: 401 });
+  try {
+    const { jobId } = await context.params;
+    return Response.json(await antflyIngestion.cancelJob(jobId));
+  } catch (error) {
+    const status = error instanceof AntflyIngestionError ? error.status : 502;
+    return Response.json({ error: error instanceof Error ? error.message : "Job cancellation failed." }, { status });
+  }
+}

app/api/admin/ingestion/route.ts

@@ -0,0 +1,28 @@
+import { isAdminRequest } from "@/lib/ingestion/auth";
+import { antflyIngestion, AntflyIngestionError } from "@/lib/ingestion/client";
+
+export const dynamic = "force-dynamic";
+
+function failure(error: unknown) {
+  const status = error instanceof AntflyIngestionError ? error.status : 502;
+  const message = error instanceof Error ? error.message : "Antfly ingestion request failed.";
+  return Response.json({ error: message }, { status });
+}
+
+export async function GET(request: Request) {
+  if (!isAdminRequest(request)) return Response.json({ error: "Unauthorized" }, { status: 401 });
+  try {
+    return Response.json(await antflyIngestion.overview());
+  } catch (error) {
+    return failure(error);
+  }
+}
+
+export async function POST(request: Request) {
+  if (!isAdminRequest(request)) return Response.json({ error: "Unauthorized" }, { status: 401 });
+  try {
+    return Response.json(await antflyIngestion.createSource(await request.json()), { status: 201 });
+  } catch (error) {
+    return failure(error);
+  }
+}

app/api/admin/ingestion/sources/[sourceId]/jobs/route.ts

@@ -0,0 +1,13 @@
+import { isAdminRequest } from "@/lib/ingestion/auth";
+import { antflyIngestion, AntflyIngestionError } from "@/lib/ingestion/client";
+
+export async function POST(request: Request, context: { params: Promise<{ sourceId: string }> }) {
+  if (!isAdminRequest(request)) return Response.json({ error: "Unauthorized" }, { status: 401 });
+  try {
+    const { sourceId } = await context.params;
+    return Response.json(await antflyIngestion.runSource(sourceId), { status: 202 });
+  } catch (error) {
+    const status = error instanceof AntflyIngestionError ? error.status : 502;
+    return Response.json({ error: error instanceof Error ? error.message : "Job submission failed." }, { status });
+  }
+}

app/globals.css

@@ -182,10 +182,28 @@
   border-bottom: 0;
 }
 
+.source-auth, .connector-grid article, .source-list { border: 1px solid #dbe4eb; border-radius: 1rem; background: white; }
+.source-auth { display: flex; justify-content: space-between; gap: 2rem; padding: 1.25rem; }
+.source-auth h2, .connector-grid h3 { margin: .25rem 0; }
+.source-auth p, .connector-grid p, .source-list p { color: #64748b; }
+.source-auth form { display: flex; align-items: center; gap: .5rem; }
+.source-auth input { min-width: 16rem; padding: .7rem; border: 1px solid #cbd5e1; border-radius: .6rem; }
+.source-auth button, .source-list button { padding: .7rem 1rem; border: 0; border-radius: .6rem; background: #0f766e; color: white; cursor: pointer; }
+.connector-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 1rem; }
+.connector-grid article { padding: 1.25rem; }
+.connector-grid small { color: #0f766e; }
+.source-list { overflow: hidden; }
+.source-list article { display: flex; align-items: center; justify-content: space-between; gap: 1rem; padding: 1rem 1.25rem; border-bottom: 1px solid #e2e8f0; }
+.source-list article:last-child { border-bottom: 0; }
+.source-list p { margin: .25rem 0 0; }
+.source-empty, .source-error { padding: 1rem 1.25rem; }
+.source-error { width: min(1120px, 100%); margin: 1rem auto 0; border-radius: .75rem; background: #fff7ed; color: #c2410c; }
+
 @media (max-width: 850px) {
   .admin-card-grid {
     grid-template-columns: repeat(2, minmax(0, 1fr));
   }
+  .connector-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
 }
 
 @media (max-width: 560px) {
@@ -198,6 +216,9 @@
   .admin-card-grid {
     grid-template-columns: 1fr;
   }
+  .source-auth, .source-auth form { align-items: stretch; flex-direction: column; }
+  .source-auth input { min-width: 0; }
+  .connector-grid { grid-template-columns: 1fr; }
 }
 
 * {

components/knowledge-sources.tsx

@@ -0,0 +1,89 @@
+"use client";
+
+import { useState } from "react";
+import type { IngestionOverview } from "@/lib/ingestion/types";
+
+const connectorCopy = [
+  ["sitemap", "Sitemap", "Discover and synchronize documentation URLs."],
+  ["github", "GitHub", "Synchronize selected repositories, branches, and paths."],
+  ["s3", "S3", "Import and refresh objects from an S3-compatible bucket."],
+  ["upload", "Files", "Submit files to Antfly for extraction and indexing."],
+] as const;
+
+export function KnowledgeSources() {
+  const [token, setToken] = useState("");
+  const [overview, setOverview] = useState<IngestionOverview>();
+  const [error, setError] = useState<string>();
+  const [loading, setLoading] = useState(false);
+
+  async function connect() {
+    setLoading(true);
+    setError(undefined);
+    try {
+      const response = await fetch("/api/admin/ingestion", {
+        headers: { Authorization: `Bearer ${token}` },
+      });
+      const payload = await response.json();
+      if (!response.ok) throw new Error(payload.error || "Unable to load ingestion sources.");
+      sessionStorage.setItem("knowledge-admin-token", token);
+      setOverview(payload);
+    } catch (caught) {
+      setOverview(undefined);
+      setError(caught instanceof Error ? caught.message : "Unable to connect.");
+    } finally {
+      setLoading(false);
+    }
+  }
+
+  async function run(sourceId: string) {
+    setError(undefined);
+    const response = await fetch(`/api/admin/ingestion/sources/${encodeURIComponent(sourceId)}/jobs`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${token}` },
+    });
+    const payload = await response.json();
+    if (!response.ok) return setError(payload.error || "Unable to start job.");
+    await connect();
+  }
+
+  const capabilities = new Map(overview?.capabilities.map((item) => [item.kind, item]));
+
+  return (
+    <>
+      <section className="source-auth" aria-labelledby="source-access-title">
+        <div>
+          <span>Protected control plane</span>
+          <h2 id="source-access-title">Connect to Antfly ingestion</h2>
+          <p>The token stays in this browser session. Antfly credentials remain server-only.</p>
+        </div>
+        <form onSubmit={(event) => { event.preventDefault(); void connect(); }}>
+          <input aria-label="Admin token" onChange={(event) => setToken(event.target.value)} placeholder="Admin token" type="password" value={token} />
+          <button disabled={!token || loading}>{loading ? "Connecting…" : "Connect"}</button>
+        </form>
+      </section>
+
+      {error && <p className="source-error" role="alert">{error}</p>}
+
+      <section aria-labelledby="connectors-title">
+        <div className="admin-section-heading"><div><span>Connectors</span><h2 id="connectors-title">Bring your knowledge</h2></div></div>
+        <div className="connector-grid">
+          {connectorCopy.map(([kind, label, description]) => {
+            const capability = capabilities.get(kind);
+            return <article key={kind}><div className={`admin-status ${capability?.available ? "ready" : ""}`}>{capability?.available ? "Available" : "Backend required"}</div><h3>{label}</h3><p>{description}</p><small>Executed as a durable Antfly job.</small></article>;
+          })}
+        </div>
+      </section>
+
+      <section aria-labelledby="sources-title">
+        <div className="admin-section-heading"><div><span>Antfly</span><h2 id="sources-title">Configured sources</h2></div><p>{overview ? `${overview.sources.length} sources` : "Connect to load sources"}</p></div>
+        <div className="source-list">
+          {overview?.sources.map((source) => <article key={source.id}><div><strong>{source.name}</strong><p>{source.connector} · {source.status}</p></div><button onClick={() => void run(source.id)}>Sync now</button></article>)}
+          {overview && overview.sources.length === 0 && <p className="source-empty">No sources configured in Antfly yet.</p>}
+          {!overview && <p className="source-empty">Source definitions and job state are loaded directly from Antfly.</p>}
+        </div>
+      </section>
+
+      {overview && <section aria-labelledby="jobs-title"><div className="admin-section-heading"><div><span>Durable execution</span><h2 id="jobs-title">Recent jobs</h2></div></div><div className="source-list">{overview.jobs.map((job) => <article key={job.id}><div><strong>{job.connector} job</strong><p>{job.status}{job.progress?.message ? ` · ${job.progress.message}` : ""}</p></div><code>{job.id}</code></article>)}{overview.jobs.length === 0 && <p className="source-empty">No ingestion jobs yet.</p>}</div></section>}
+    </>
+  );
+}

docs/ANTFLY_CONNECTOR_IMPLEMENTATION.md

@@ -0,0 +1,134 @@
+# Antfly knowledge connector implementation brief
+
+This brief defines the Antfly backend work required by the Knowledge Support
+template. Connector execution is an Antfly responsibility. The Next.js template
+only creates source definitions, submits jobs, and displays backend state.
+
+## Scope
+
+Build two connectors:
+
+1. Sitemap synchronization
+2. GitHub repository synchronization
+
+S3 ingestion already exists and should not be rebuilt. It should be adapted to
+the common source/job contract and used as the reference implementation and
+conformance fixture.
+
+## Shared backend foundation
+
+Implement this foundation once before connector-specific work:
+
+- Durable `KnowledgeSource` records scoped to an Antfly instance and table.
+- Encrypted connector secrets referenced by ID; API responses never return a
+  token, private key, or resolved secret.
+- Durable, resumable `IngestionJob` records with queued, running, succeeded,
+  failed, and cancelled states.
+- An idempotency key for source creation and job submission.
+- One active synchronization per source, with explicit coalescing or conflict
+  behavior for duplicate requests.
+- Checkpoints that allow retries without restarting completed work.
+- Stable document identity derived from source ID and canonical remote identity.
+- Reconciliation: documents absent from a successful full scan are deleted or
+  tombstoned from the target knowledge collection.
+- A connector interface that emits normalized source documents into Antfly's
+  existing extraction, artifact, chunking, embedding, and indexing pipeline.
+- Structured progress, warnings, retryable errors, terminal errors, counters,
+  timestamps, and trace IDs.
+- Per-instance concurrency, bandwidth, document-count, and byte limits.
+
+The public integration contract consumed by the template is documented in
+[ANTFLY_INGESTION_CONTRACT.md](ANTFLY_INGESTION_CONTRACT.md).
+
+## Sitemap connector
+
+### Source configuration
+
+- Sitemap URL
+- Target table or knowledge collection
+- Optional include and exclude path patterns
+- Optional maximum URLs and crawl depth
+- Optional schedule
+- Rendering policy: static HTML initially; JavaScript rendering is a separate
+  capability and should not be implied
+
+### Job behavior
+
+1. Validate HTTP/S URLs and reject credentials, loopback, link-local, private,
+   metadata-service, and disallowed redirect targets.
+2. Fetch sitemap XML with timeouts, compressed-response limits, conditional
+   requests, and bounded retries.
+3. Support both `urlset` and nested `sitemapindex` documents with cycle
+   detection and configured limits.
+4. Canonicalize and deduplicate URLs, then apply source scope and path filters.
+5. Fetch changed pages using ETag and Last-Modified checkpoints where available.
+6. Pass response content and provenance to Antfly's extraction pipeline.
+7. Preserve title, canonical URL, source URL, content type, timestamps, checksum,
+   and source/job IDs on every normalized document.
+8. Reconcile removals only after a complete successful discovery pass. A
+   partial or failed crawl must never mass-delete indexed content.
+
+### Acceptance criteria
+
+- Nested and gzip-compressed sitemaps are supported.
+- A retry resumes from a checkpoint without duplicating documents.
+- Unchanged pages do not trigger unnecessary extraction or embedding work.
+- Changed and removed pages are reflected after a successful sync.
+- Redirects and DNS resolution cannot be used for SSRF.
+- One malformed page is reported without necessarily failing the whole job.
+
+## GitHub connector
+
+### Source configuration
+
+- GitHub repository owner/name and branch or tag
+- GitHub App installation or secret reference; a fine-grained token may be a
+  development fallback
+- Include/exclude glob patterns
+- Supported text/document extensions and maximum blob size
+- Optional schedule and webhook synchronization
+- Target table or knowledge collection
+
+### Job behavior
+
+1. Resolve the configured ref to an immutable commit SHA.
+2. Enumerate the repository tree using GitHub's API, respecting pagination,
+   rate limits, retries, and secondary-rate-limit backoff.
+3. Filter paths before downloading blobs. Skip binaries, symlinks, submodules,
+   generated/vendor directories, and oversized files unless explicitly enabled.
+4. Fetch supported blobs and pass them to Antfly extraction with repository,
+   ref, commit, path, blob SHA, and public source URL provenance.
+5. Use blob SHA plus extraction configuration as the change detector.
+6. Checkpoint the commit and completed paths for resumability.
+7. Reconcile renamed and deleted paths only after a complete successful tree
+   traversal.
+8. For webhooks, verify signatures and enqueue a normal durable sync job rather
+   than processing repository content in the webhook request.
+
+### Acceptance criteria
+
+- Public and private repositories work through least-privilege credentials.
+- Full sync, incremental commit sync, rename, deletion, force-push, and default
+  branch change cases are covered.
+- Rate limiting moves the job into a visible retry state without losing progress.
+- Secret values are redacted from API responses, events, errors, and logs.
+- Re-running the same commit is idempotent.
+
+## Delivery sequence
+
+1. Finalize the source/job OpenAPI schemas and TypeScript/Go SDK generation.
+2. Adapt existing S3 ingestion to the contract and write connector conformance
+   tests against it.
+3. Implement Sitemap with static HTML extraction and full reconciliation.
+4. Implement GitHub scheduled/full sync using a GitHub App.
+5. Add GitHub incremental sync and webhook enqueueing.
+6. Enable capability discovery in Antfly Cloud, then remove the corresponding
+   `Backend required` states from the template through live discovery.
+
+## Definition of done
+
+Each connector ships with OpenAPI documentation, SDK methods, unit tests,
+integration tests using local fixtures, failure/retry tests, security tests,
+metrics, operator documentation, and an end-to-end test exercised through the
+Knowledge Support template. No connector worker or durable job state runs in
+Vercel.