diff --git a/.env.example b/.env.example index b6100af..d5f0d4d 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,14 @@ ANTFLY_INFERENCE_URL=https://platform.antfly.io/cloud/v1/INSTANCE_ID/ai/v1 # Paste only the token value; do not include "Bearer ". ANTFLY_API_KEY= +# Antfly-native connector and durable ingestion-job API. Keep this write-capable +# credential separate from the read-only support-agent key above. +# Set this when the Antfly knowledge-ingestion API is enabled for your instance. +ANTFLY_INGESTION_URL= +ANTFLY_INGESTION_API_KEY= +# Protects this template's /api/admin/ingestion routes. Use a long random value. +KNOWLEDGE_ADMIN_TOKEN= + # Antfly document table and indexes ANTFLY_TABLE=docs ANTFLY_VECTOR_INDEX=document_vectors diff --git a/README.md b/README.md index a2936bc..5b0d9f4 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,22 @@ documentation search experience, and grounded support agent with: - Reusable `⌘K` / `Ctrl+K` support command palette with an embedded chat modal - A server-only deployment-readiness dashboard at `/admin` -The current foundation provides the production support-agent path. Website, -GitHub, and file connectors; conventional search; durable conversations; -analytics; content-gap detection; and escalation adapters are the next staged -modules. Their product boundary and release gates are maintained in the -`knowledge-support` entry in the Antfly template library. +The current foundation provides the production support-agent path and an +Antfly-native ingestion control plane at `/admin/sources`. Sitemap, GitHub, S3, +and file connectors execute as durable Antfly backend jobs. Conventional +search, durable conversations, analytics, content-gap detection, and escalation +adapters are the next staged modules. Their product boundary and release gates +are maintained in the `knowledge-support` entry in the Antfly template library. The browser never receives Antfly or model-provider credentials. Retrieval and generation run in server-side Next.js routes. +The template does not crawl, parse, or queue ingestion work in Vercel. See the +[Antfly ingestion contract](docs/ANTFLY_INGESTION_CONTRACT.md) and +[connector implementation brief](docs/ANTFLY_CONNECTOR_IMPLEMENTATION.md). + +For the component-level product gap and recommended 80% milestone, see the +[Kapa React comparison](docs/KAPA_REACT_COMPARISON.md). + ## Support command palette The reference page mounts `SupportCommandPalette` once alongside the full-page diff --git a/app/admin/page.tsx b/app/admin/page.tsx index ca8c77f..ece4fd6 100644 --- a/app/admin/page.tsx +++ b/app/admin/page.tsx @@ -6,7 +6,7 @@ const surfaces = [ { title: "Knowledge sources", description: "Website, sitemap, GitHub, and file synchronization.", - status: "Foundation", + status: "Antfly jobs", }, { title: "Search and answers", diff --git a/app/admin/sources/page.tsx b/app/admin/sources/page.tsx new file mode 100644 index 0000000..01aeff6 --- /dev/null +++ b/app/admin/sources/page.tsx @@ -0,0 +1,6 @@ +import Link from "next/link"; +import { KnowledgeSources } from "@/components/knowledge-sources"; + +export default function SourcesPage() { + return
Knowledge operations

Sources

Configure knowledge sources and monitor the Antfly jobs that ingest them.

Admin home
; +} diff --git a/app/api/admin/ingestion/jobs/[jobId]/cancel/route.ts b/app/api/admin/ingestion/jobs/[jobId]/cancel/route.ts new file mode 100644 index 0000000..37dff84 --- /dev/null +++ b/app/api/admin/ingestion/jobs/[jobId]/cancel/route.ts @@ -0,0 +1,13 @@ +import { isAdminRequest } from "@/lib/ingestion/auth"; +import { antflyIngestion, AntflyIngestionError } from "@/lib/ingestion/client"; + +export async function POST(request: Request, context: { params: Promise<{ jobId: string }> }) { + if (!isAdminRequest(request)) return Response.json({ error: "Unauthorized" }, { status: 401 }); + try { + const { jobId } = await context.params; + return Response.json(await antflyIngestion.cancelJob(jobId)); + } catch (error) { + const status = error instanceof AntflyIngestionError ? error.status : 502; + return Response.json({ error: error instanceof Error ? error.message : "Job cancellation failed." }, { status }); + } +} diff --git a/app/api/admin/ingestion/route.ts b/app/api/admin/ingestion/route.ts new file mode 100644 index 0000000..9bf5b26 --- /dev/null +++ b/app/api/admin/ingestion/route.ts @@ -0,0 +1,28 @@ +import { isAdminRequest } from "@/lib/ingestion/auth"; +import { antflyIngestion, AntflyIngestionError } from "@/lib/ingestion/client"; + +export const dynamic = "force-dynamic"; + +function failure(error: unknown) { + const status = error instanceof AntflyIngestionError ? error.status : 502; + const message = error instanceof Error ? error.message : "Antfly ingestion request failed."; + return Response.json({ error: message }, { status }); +} + +export async function GET(request: Request) { + if (!isAdminRequest(request)) return Response.json({ error: "Unauthorized" }, { status: 401 }); + try { + return Response.json(await antflyIngestion.overview()); + } catch (error) { + return failure(error); + } +} + +export async function POST(request: Request) { + if (!isAdminRequest(request)) return Response.json({ error: "Unauthorized" }, { status: 401 }); + try { + return Response.json(await antflyIngestion.createSource(await request.json()), { status: 201 }); + } catch (error) { + return failure(error); + } +} diff --git a/app/api/admin/ingestion/sources/[sourceId]/jobs/route.ts b/app/api/admin/ingestion/sources/[sourceId]/jobs/route.ts new file mode 100644 index 0000000..d098ccf --- /dev/null +++ b/app/api/admin/ingestion/sources/[sourceId]/jobs/route.ts @@ -0,0 +1,13 @@ +import { isAdminRequest } from "@/lib/ingestion/auth"; +import { antflyIngestion, AntflyIngestionError } from "@/lib/ingestion/client"; + +export async function POST(request: Request, context: { params: Promise<{ sourceId: string }> }) { + if (!isAdminRequest(request)) return Response.json({ error: "Unauthorized" }, { status: 401 }); + try { + const { sourceId } = await context.params; + return Response.json(await antflyIngestion.runSource(sourceId), { status: 202 }); + } catch (error) { + const status = error instanceof AntflyIngestionError ? error.status : 502; + return Response.json({ error: error instanceof Error ? error.message : "Job submission failed." }, { status }); + } +} diff --git a/app/globals.css b/app/globals.css index 673c665..c01cca1 100644 --- a/app/globals.css +++ b/app/globals.css @@ -182,10 +182,28 @@ border-bottom: 0; } +.source-auth, .connector-grid article, .source-list { border: 1px solid #dbe4eb; border-radius: 1rem; background: white; } +.source-auth { display: flex; justify-content: space-between; gap: 2rem; padding: 1.25rem; } +.source-auth h2, .connector-grid h3 { margin: .25rem 0; } +.source-auth p, .connector-grid p, .source-list p { color: #64748b; } +.source-auth form { display: flex; align-items: center; gap: .5rem; } +.source-auth input { min-width: 16rem; padding: .7rem; border: 1px solid #cbd5e1; border-radius: .6rem; } +.source-auth button, .source-list button { padding: .7rem 1rem; border: 0; border-radius: .6rem; background: #0f766e; color: white; cursor: pointer; } +.connector-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 1rem; } +.connector-grid article { padding: 1.25rem; } +.connector-grid small { color: #0f766e; } +.source-list { overflow: hidden; } +.source-list article { display: flex; align-items: center; justify-content: space-between; gap: 1rem; padding: 1rem 1.25rem; border-bottom: 1px solid #e2e8f0; } +.source-list article:last-child { border-bottom: 0; } +.source-list p { margin: .25rem 0 0; } +.source-empty, .source-error { padding: 1rem 1.25rem; } +.source-error { width: min(1120px, 100%); margin: 1rem auto 0; border-radius: .75rem; background: #fff7ed; color: #c2410c; } + @media (max-width: 850px) { .admin-card-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .connector-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } } @media (max-width: 560px) { @@ -198,6 +216,9 @@ .admin-card-grid { grid-template-columns: 1fr; } + .source-auth, .source-auth form { align-items: stretch; flex-direction: column; } + .source-auth input { min-width: 0; } + .connector-grid { grid-template-columns: 1fr; } } * { diff --git a/components/knowledge-sources.tsx b/components/knowledge-sources.tsx new file mode 100644 index 0000000..5fe39f1 --- /dev/null +++ b/components/knowledge-sources.tsx @@ -0,0 +1,89 @@ +"use client"; + +import { useState } from "react"; +import type { IngestionOverview } from "@/lib/ingestion/types"; + +const connectorCopy = [ + ["sitemap", "Sitemap", "Discover and synchronize documentation URLs."], + ["github", "GitHub", "Synchronize selected repositories, branches, and paths."], + ["s3", "S3", "Import and refresh objects from an S3-compatible bucket."], + ["upload", "Files", "Submit files to Antfly for extraction and indexing."], +] as const; + +export function KnowledgeSources() { + const [token, setToken] = useState(""); + const [overview, setOverview] = useState(); + const [error, setError] = useState(); + const [loading, setLoading] = useState(false); + + async function connect() { + setLoading(true); + setError(undefined); + try { + const response = await fetch("/api/admin/ingestion", { + headers: { Authorization: `Bearer ${token}` }, + }); + const payload = await response.json(); + if (!response.ok) throw new Error(payload.error || "Unable to load ingestion sources."); + sessionStorage.setItem("knowledge-admin-token", token); + setOverview(payload); + } catch (caught) { + setOverview(undefined); + setError(caught instanceof Error ? caught.message : "Unable to connect."); + } finally { + setLoading(false); + } + } + + async function run(sourceId: string) { + setError(undefined); + const response = await fetch(`/api/admin/ingestion/sources/${encodeURIComponent(sourceId)}/jobs`, { + method: "POST", + headers: { Authorization: `Bearer ${token}` }, + }); + const payload = await response.json(); + if (!response.ok) return setError(payload.error || "Unable to start job."); + await connect(); + } + + const capabilities = new Map(overview?.capabilities.map((item) => [item.kind, item])); + + return ( + <> +
+
+ Protected control plane +

Connect to Antfly ingestion

+

The token stays in this browser session. Antfly credentials remain server-only.

+
+
{ event.preventDefault(); void connect(); }}> + setToken(event.target.value)} placeholder="Admin token" type="password" value={token} /> + +
+
+ + {error &&

{error}

} + +
+
Connectors

Bring your knowledge

+
+ {connectorCopy.map(([kind, label, description]) => { + const capability = capabilities.get(kind); + return
{capability?.available ? "Available" : "Backend required"}

{label}

{description}

Executed as a durable Antfly job.
; + })} +
+
+ +
+
Antfly

Configured sources

{overview ? `${overview.sources.length} sources` : "Connect to load sources"}

+
+ {overview?.sources.map((source) =>
{source.name}

{source.connector} · {source.status}

)} + {overview && overview.sources.length === 0 &&

No sources configured in Antfly yet.

} + {!overview &&

Source definitions and job state are loaded directly from Antfly.

} +
+
+ + {overview &&
Durable execution

Recent jobs

{overview.jobs.map((job) =>
{job.connector} job

{job.status}{job.progress?.message ? ` · ${job.progress.message}` : ""}

{job.id}
)}{overview.jobs.length === 0 &&

No ingestion jobs yet.

}
} + + ); +} diff --git a/docs/ANTFLY_CONNECTOR_IMPLEMENTATION.md b/docs/ANTFLY_CONNECTOR_IMPLEMENTATION.md new file mode 100644 index 0000000..6076088 --- /dev/null +++ b/docs/ANTFLY_CONNECTOR_IMPLEMENTATION.md @@ -0,0 +1,134 @@ +# Antfly knowledge connector implementation brief + +This brief defines the Antfly backend work required by the Knowledge Support +template. Connector execution is an Antfly responsibility. The Next.js template +only creates source definitions, submits jobs, and displays backend state. + +## Scope + +Build two connectors: + +1. Sitemap synchronization +2. GitHub repository synchronization + +S3 ingestion already exists and should not be rebuilt. It should be adapted to +the common source/job contract and used as the reference implementation and +conformance fixture. + +## Shared backend foundation + +Implement this foundation once before connector-specific work: + +- Durable `KnowledgeSource` records scoped to an Antfly instance and table. +- Encrypted connector secrets referenced by ID; API responses never return a + token, private key, or resolved secret. +- Durable, resumable `IngestionJob` records with queued, running, succeeded, + failed, and cancelled states. +- An idempotency key for source creation and job submission. +- One active synchronization per source, with explicit coalescing or conflict + behavior for duplicate requests. +- Checkpoints that allow retries without restarting completed work. +- Stable document identity derived from source ID and canonical remote identity. +- Reconciliation: documents absent from a successful full scan are deleted or + tombstoned from the target knowledge collection. +- A connector interface that emits normalized source documents into Antfly's + existing extraction, artifact, chunking, embedding, and indexing pipeline. +- Structured progress, warnings, retryable errors, terminal errors, counters, + timestamps, and trace IDs. +- Per-instance concurrency, bandwidth, document-count, and byte limits. + +The public integration contract consumed by the template is documented in +[ANTFLY_INGESTION_CONTRACT.md](ANTFLY_INGESTION_CONTRACT.md). + +## Sitemap connector + +### Source configuration + +- Sitemap URL +- Target table or knowledge collection +- Optional include and exclude path patterns +- Optional maximum URLs and crawl depth +- Optional schedule +- Rendering policy: static HTML initially; JavaScript rendering is a separate + capability and should not be implied + +### Job behavior + +1. Validate HTTP/S URLs and reject credentials, loopback, link-local, private, + metadata-service, and disallowed redirect targets. +2. Fetch sitemap XML with timeouts, compressed-response limits, conditional + requests, and bounded retries. +3. Support both `urlset` and nested `sitemapindex` documents with cycle + detection and configured limits. +4. Canonicalize and deduplicate URLs, then apply source scope and path filters. +5. Fetch changed pages using ETag and Last-Modified checkpoints where available. +6. Pass response content and provenance to Antfly's extraction pipeline. +7. Preserve title, canonical URL, source URL, content type, timestamps, checksum, + and source/job IDs on every normalized document. +8. Reconcile removals only after a complete successful discovery pass. A + partial or failed crawl must never mass-delete indexed content. + +### Acceptance criteria + +- Nested and gzip-compressed sitemaps are supported. +- A retry resumes from a checkpoint without duplicating documents. +- Unchanged pages do not trigger unnecessary extraction or embedding work. +- Changed and removed pages are reflected after a successful sync. +- Redirects and DNS resolution cannot be used for SSRF. +- One malformed page is reported without necessarily failing the whole job. + +## GitHub connector + +### Source configuration + +- GitHub repository owner/name and branch or tag +- GitHub App installation or secret reference; a fine-grained token may be a + development fallback +- Include/exclude glob patterns +- Supported text/document extensions and maximum blob size +- Optional schedule and webhook synchronization +- Target table or knowledge collection + +### Job behavior + +1. Resolve the configured ref to an immutable commit SHA. +2. Enumerate the repository tree using GitHub's API, respecting pagination, + rate limits, retries, and secondary-rate-limit backoff. +3. Filter paths before downloading blobs. Skip binaries, symlinks, submodules, + generated/vendor directories, and oversized files unless explicitly enabled. +4. Fetch supported blobs and pass them to Antfly extraction with repository, + ref, commit, path, blob SHA, and public source URL provenance. +5. Use blob SHA plus extraction configuration as the change detector. +6. Checkpoint the commit and completed paths for resumability. +7. Reconcile renamed and deleted paths only after a complete successful tree + traversal. +8. For webhooks, verify signatures and enqueue a normal durable sync job rather + than processing repository content in the webhook request. + +### Acceptance criteria + +- Public and private repositories work through least-privilege credentials. +- Full sync, incremental commit sync, rename, deletion, force-push, and default + branch change cases are covered. +- Rate limiting moves the job into a visible retry state without losing progress. +- Secret values are redacted from API responses, events, errors, and logs. +- Re-running the same commit is idempotent. + +## Delivery sequence + +1. Finalize the source/job OpenAPI schemas and TypeScript/Go SDK generation. +2. Adapt existing S3 ingestion to the contract and write connector conformance + tests against it. +3. Implement Sitemap with static HTML extraction and full reconciliation. +4. Implement GitHub scheduled/full sync using a GitHub App. +5. Add GitHub incremental sync and webhook enqueueing. +6. Enable capability discovery in Antfly Cloud, then remove the corresponding + `Backend required` states from the template through live discovery. + +## Definition of done + +Each connector ships with OpenAPI documentation, SDK methods, unit tests, +integration tests using local fixtures, failure/retry tests, security tests, +metrics, operator documentation, and an end-to-end test exercised through the +Knowledge Support template. No connector worker or durable job state runs in +Vercel. diff --git a/docs/ANTFLY_INGESTION_CONTRACT.md b/docs/ANTFLY_INGESTION_CONTRACT.md new file mode 100644 index 0000000..864566a --- /dev/null +++ b/docs/ANTFLY_INGESTION_CONTRACT.md @@ -0,0 +1,28 @@ +# Antfly knowledge ingestion contract + +This template is a control plane for ingestion. Antfly owns connector execution, +credentials, durable state, retries, extraction, chunking, deletion reconciliation, +and indexing. Vercel never crawls a site or runs a synchronization queue. + +See [ANTFLY_CONNECTOR_IMPLEMENTATION.md](ANTFLY_CONNECTOR_IMPLEMENTATION.md) +for the Sitemap and GitHub backend work breakdown. Existing S3 ingestion is the +reference connector; it is not part of that new implementation scope. + +`ANTFLY_INGESTION_URL` points to the versioned Antfly service implementing the +following integration contract. This general connector API is a backend +dependency and is not yet part of Antfly's current public OpenAPI surface: + +| Method | Path | Purpose | +| --- | --- | --- | +| `GET` | `/overview` | Connector capabilities, sources, and recent jobs | +| `POST` | `/sources` | Create a source definition | +| `POST` | `/sources/{sourceId}/jobs` | Start a durable synchronization job | +| `POST` | `/jobs/{jobId}/cancel` | Request cancellation | + +The TypeScript wire types are in `lib/ingestion/types.ts`. Connector-specific +configuration is described by each capability's JSON schema so Antfly can add +Sitemap, GitHub, S3, and future connectors without a new template API. + +The ingestion credential must be distinct from `ANTFLY_API_KEY`, which remains a +read-only retrieval credential. `/api/admin/ingestion/*` requires +`KNOWLEDGE_ADMIN_TOKEN` and never returns either Antfly credential. diff --git a/docs/KAPA_REACT_COMPARISON.md b/docs/KAPA_REACT_COMPARISON.md new file mode 100644 index 0000000..2414f3f --- /dev/null +++ b/docs/KAPA_REACT_COMPARISON.md @@ -0,0 +1,75 @@ +# React support-agent comparison: Antfly and Kapa + +Assessment date: August 8, 2026. + +Kapa's React Agent SDK is an application integration layer over Kapa's hosted +knowledge and agent platform. Antfly currently has two complementary pieces: + +- this Next.js/Vercel template, which is a complete, deployable support site and + server integration; and +- `@antfly/react-antfly`, which provides search/RAG components plus the newer + `SupportProvider`, `SupportChat`, and `useSupportAgent` API. + +These are not exact substitutes. The template is closer to a starter +application; Kapa's package is closer to a polished agent SDK. + +## Capability map + +| Capability | Antfly today | Kapa React today | Priority | +| --- | --- | --- | --- | +| Server-held platform credential | Yes; Next.js routes own Antfly/model keys | Yes; server creates a short-lived session token | Keep; add scoped sessions | +| Provider and complete chat | `SupportProvider` + `SupportChat` | `AgentProvider` + `AgentChat` | Near parity | +| Grounded KB answers and citations | Yes | Yes, through built-in server-side KB search | Near parity | +| Starter prompts and branding | Yes | Yes, with a broader branding configuration | Near parity | +| Feedback | Yes; application-owned endpoint and callbacks | Available through Kapa's platform/integrations | Antfly strength/control | +| Streaming | Antfly RAG hooks stream, but `SupportChat` and this template currently await a complete answer | Built into `AgentChat` | P0 gap | +| Conversation state | Provider state for the current mount | Provider owns messages, streaming, sessions, and approvals | P0 hardening | +| Durable thread history | Not in support components/template | Optional `AgentThreadHistory` and resumable conversations | P1 gap | +| Slide-in panel | Template has a command-palette modal; no reusable SDK panel primitive | `AgentPanel` | P1 gap | +| Theme API | CSS/application customization | Typed accent, scheme, font, size, and radius configuration | P1 gap | +| Custom tools and approvals | Not in support components | Client tools, context, approval UI, and custom rendering | P2 for support; P0 only for in-product agents | +| Tool-call inspection | Not in support chat | Expandable arguments, responses, and sources | P2 | +| Source-group filtering | Can be implemented in the server retrieval route, but no public provider prop | `sourceGroupIdsInclude` | P1 for multi-product KBs | +| Analytics events | Feedback and server metrics exist; no unified React event API | Typed lifecycle events | P1 gap | +| Search UI primitives | Full-text/vector search, facets, results, autosuggest, RAG hooks | Agent SDK focuses on agent chat | Antfly strength | +| Hosting and ownership | Customer-owned app and Antfly deployment | Kapa-managed platform and sessions | Antfly differentiator | + +## What constitutes practical 80% parity + +The next React milestone should not copy Kapa's entire tool ecosystem. For a +customer building a documentation and support assistant, the highest-value work +is: + +1. Stream the support transport and render partial Markdown safely. +2. Add server-created, short-lived Antfly support sessions so browsers never + receive durable credentials and conversations have a stable owner. +3. Persist and resume threads, with a small `SupportThreadHistory` component. +4. Add a reusable `SupportPanel` while retaining the current full-page and + command-palette surfaces. +5. Provide a typed theme configuration and consistent event callbacks. +6. Add source-group filters to provider/session configuration. + +That delivers the experience most documentation customers notice. Custom +client tools, approval flows, and rich tool renderers should follow as a +separate in-product-agent milestone rather than blocking support-agent parity. + +## Architectural difference + +Kapa's quickstart sends the browser a short-lived agent session token; its +provider then talks to Kapa and owns streaming and conversation state. Antfly's +current template keeps the entire retrieval/generation call behind an +application-owned server route. That is secure and highly controllable, but it +makes each customer application responsible for more protocol behavior. + +The recommended Antfly end state combines both approaches: Antfly owns durable +knowledge jobs, scoped support sessions, retrieval, and thread state; +`@antfly/react-antfly` owns the browser protocol and reusable UI; this template +provides the deployable reference application and admin experience. + +## Sources + +- [Kapa React quickstart](https://docs.kapa.ai/dev/agent/quickstart-react) +- [Kapa AgentProvider](https://docs.kapa.ai/dev/agent/react/agent-provider) +- [Kapa AgentChat](https://docs.kapa.ai/dev/agent/react/agent-chat) +- [Kapa platform overview](https://docs.kapa.ai/) +- [React Antfly repository](https://github.com/antflydb/react-antfly) diff --git a/lib/ingestion/auth.ts b/lib/ingestion/auth.ts new file mode 100644 index 0000000..eea8469 --- /dev/null +++ b/lib/ingestion/auth.ts @@ -0,0 +1,10 @@ +import { timingSafeEqual } from "node:crypto"; + +export function isAdminRequest(request: Request): boolean { + const expected = process.env.KNOWLEDGE_ADMIN_TOKEN?.trim(); + const supplied = request.headers.get("authorization")?.replace(/^Bearer\s+/i, ""); + if (!expected || !supplied) return false; + const left = Buffer.from(expected); + const right = Buffer.from(supplied); + return left.length === right.length && timingSafeEqual(left, right); +} diff --git a/lib/ingestion/client.ts b/lib/ingestion/client.ts new file mode 100644 index 0000000..9e3a550 --- /dev/null +++ b/lib/ingestion/client.ts @@ -0,0 +1,60 @@ +import type { + CreateSourceInput, + IngestionJob, + IngestionOverview, + KnowledgeSource, +} from "@/lib/ingestion/types"; + +type RequestOptions = { method?: "GET" | "POST" | "DELETE"; body?: unknown }; + +export class AntflyIngestionError extends Error { + constructor(message: string, readonly status: number) { + super(message); + this.name = "AntflyIngestionError"; + } +} + +function settings() { + const baseUrl = process.env.ANTFLY_INGESTION_URL?.trim().replace(/\/$/, ""); + const apiKey = process.env.ANTFLY_INGESTION_API_KEY?.trim(); + if (!baseUrl || !apiKey) { + throw new AntflyIngestionError( + "Antfly ingestion is not configured. Set ANTFLY_INGESTION_URL and ANTFLY_INGESTION_API_KEY.", + 503, + ); + } + return { baseUrl, apiKey }; +} + +async function request(path: string, options: RequestOptions = {}): Promise { + const { baseUrl, apiKey } = settings(); + const response = await fetch(`${baseUrl}${path}`, { + method: options.method || "GET", + headers: { + Accept: "application/json", + Authorization: `Bearer ${apiKey}`, + ...(options.body ? { "Content-Type": "application/json" } : {}), + }, + body: options.body ? JSON.stringify(options.body) : undefined, + cache: "no-store", + signal: AbortSignal.timeout(20_000), + }); + if (!response.ok) { + const detail = await response.text().catch(() => ""); + throw new AntflyIngestionError( + detail || `Antfly ingestion request failed (${response.status}).`, + response.status, + ); + } + return response.json() as Promise; +} + +export const antflyIngestion = { + overview: () => request("/overview"), + createSource: (input: CreateSourceInput) => + request("/sources", { method: "POST", body: input }), + runSource: (sourceId: string) => + request(`/sources/${encodeURIComponent(sourceId)}/jobs`, { method: "POST" }), + cancelJob: (jobId: string) => + request(`/jobs/${encodeURIComponent(jobId)}/cancel`, { method: "POST" }), +}; diff --git a/lib/ingestion/types.ts b/lib/ingestion/types.ts new file mode 100644 index 0000000..1ea43d2 --- /dev/null +++ b/lib/ingestion/types.ts @@ -0,0 +1,44 @@ +export type ConnectorKind = "sitemap" | "github" | "s3" | "upload"; + +export type ConnectorCapability = { + kind: ConnectorKind; + label: string; + description: string; + available: boolean; + configurationSchema?: Record; +}; + +export type KnowledgeSource = { + id: string; + name: string; + connector: ConnectorKind; + status: "active" | "paused" | "error"; + configuration: Record; + createdAt: string; + updatedAt: string; + lastJobId?: string; +}; + +export type IngestionJob = { + id: string; + sourceId: string; + connector: ConnectorKind; + status: "queued" | "running" | "succeeded" | "failed" | "cancelled"; + createdAt: string; + updatedAt: string; + progress?: { completed: number; total?: number; message?: string }; + result?: { inserted?: number; updated?: number; deleted?: number; skipped?: number }; + error?: { code: string; message: string; retryable: boolean }; +}; + +export type CreateSourceInput = { + name: string; + connector: ConnectorKind; + configuration: Record; +}; + +export type IngestionOverview = { + capabilities: ConnectorCapability[]; + sources: KnowledgeSource[]; + jobs: IngestionJob[]; +};