diff --git a/README.md b/README.md index 2f661df..2775c12 100644 --- a/README.md +++ b/README.md @@ -352,6 +352,10 @@ same for REST workers, dashboard reads, and Python submitters; future Postgres or managed/cloud storage backends can replace the same storage boundary without changing submitter contracts. +Cloud-ready backend, artifact, worker identity, and scheduler extension +contracts are documented in +[docs/extension-contracts.md](docs/extension-contracts.md). + ## Release The release workflow builds Linux, Windows, and macOS wheels plus an sdist. PyPI publishing uses Trusted Publishing through the `pypi` GitHub environment. diff --git a/crates/farm-core/src/models.rs b/crates/farm-core/src/models.rs index f7fc1c1..7ec7e99 100644 --- a/crates/farm-core/src/models.rs +++ b/crates/farm-core/src/models.rs @@ -615,6 +615,8 @@ pub struct WorkerRegister { pub labels: HashMap, #[serde(default)] pub capacity: WorkerCapacity, + #[serde(default)] + pub identity: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -634,11 +636,23 @@ pub struct WorkerInfo { pub name: String, pub labels: HashMap, pub capacity: WorkerCapacity, + #[serde(default)] + pub identity: Option, pub state: WorkerState, pub registered_at: DateTime, pub last_seen_at: DateTime, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct WorkerIdentity { + pub provider: String, + pub subject: String, + #[serde(default)] + pub attributes: HashMap, + #[serde(default)] + pub expires_at: Option>, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WorkerLogBatch { #[serde(default)] diff --git a/crates/farm-core/src/scheduler.rs b/crates/farm-core/src/scheduler.rs index 1c65492..2309b9d 100644 --- a/crates/farm-core/src/scheduler.rs +++ b/crates/farm-core/src/scheduler.rs @@ -265,6 +265,7 @@ impl InMemoryScheduler { name: registration.name, labels: registration.labels, capacity: normalize_capacity(registration.capacity), + identity: registration.identity, state: WorkerState::Online, registered_at: now, last_seen_at: now, @@ -1782,7 +1783,7 @@ mod tests { ArtifactKind, AuditEventInput, AuditOutcome, CommandSpec, LogLevel, OpenJdAmountRequirement, OpenJdAttributeRequirement, ResourceLimitDefinition, TaskArtifact, TaskAttemptState, TaskLease, TaskLeaseRenewal, TaskRequirements, TaskStarted, - WorkerCapacity, WorkerId, WorkerInfo, WorkerLogBatch, WorkerLogInput, + WorkerCapacity, WorkerId, WorkerIdentity, WorkerInfo, WorkerLogBatch, WorkerLogInput, }; use super::*; @@ -1825,6 +1826,7 @@ mod tests { name: "local".to_string(), labels: HashMap::new(), capacity: WorkerCapacity::default(), + identity: None, }) .expect("worker should register"); @@ -1880,6 +1882,7 @@ mod tests { name: "render-node-01".to_string(), labels: HashMap::new(), capacity: WorkerCapacity { slots: 4 }, + identity: None, }) .expect("worker should register"); @@ -1896,6 +1899,36 @@ mod tests { assert_eq!(snapshot.workers[0].name, "render-node-01"); } + #[test] + fn worker_registration_preserves_optional_identity() { + let scheduler = InMemoryScheduler::default(); + let expires_at = Utc::now() + Duration::minutes(30); + let identity = WorkerIdentity { + provider: "aws-sts".to_string(), + subject: "i-0123456789abcdef0".to_string(), + attributes: HashMap::from([("region".to_string(), "us-west-2".to_string())]), + expires_at: Some(expires_at), + }; + + let worker = scheduler + .register_worker(WorkerRegister { + name: "cloud-worker".to_string(), + labels: HashMap::from([("pool".to_string(), "burst".to_string())]), + capacity: WorkerCapacity { slots: 8 }, + identity: Some(identity.clone()), + }) + .expect("worker should register"); + + assert_eq!(worker.identity.as_ref(), Some(&identity)); + let listed_worker = scheduler + .list_workers() + .expect("workers should list") + .pop() + .expect("worker should be present"); + assert_eq!(listed_worker.identity.as_ref(), Some(&identity)); + assert_eq!(listed_worker.capacity.slots, 8); + } + #[test] fn metrics_snapshot_reports_operational_counters() { let scheduler = InMemoryScheduler::default(); @@ -2025,11 +2058,18 @@ mod tests { openjd: None, }) .expect("job should submit"); + let identity = WorkerIdentity { + provider: "cloud-runner".to_string(), + subject: "worker-group/render-node-01".to_string(), + attributes: HashMap::from([("lifecycle".to_string(), "spot".to_string())]), + expires_at: Some(Utc::now() + Duration::minutes(15)), + }; let worker = scheduler .register_worker(WorkerRegister { name: "render-node-01".to_string(), labels: HashMap::new(), capacity: WorkerCapacity { slots: 2 }, + identity: Some(identity.clone()), }) .expect("worker should register"); let lease = scheduler @@ -2057,6 +2097,10 @@ mod tests { assert_eq!(restored.tasks[0].state, TaskState::Pending); assert_eq!(restored.tasks[0].stdout_tail.as_deref(), Some("stdout")); assert_eq!(reopened.list_workers().unwrap()[0].name, "render-node-01"); + assert_eq!( + reopened.list_workers().unwrap()[0].identity.as_ref(), + Some(&identity) + ); let _ = std::fs::remove_file(database_path); } @@ -2142,6 +2186,7 @@ mod tests { name: "durable-worker".to_string(), labels: HashMap::new(), capacity: WorkerCapacity::default(), + identity: None, }) .expect("worker should register"); let job = scheduler @@ -2218,6 +2263,7 @@ mod tests { name: "worker".to_string(), labels: HashMap::new(), capacity: WorkerCapacity::default(), + identity: None, }) .expect("worker should register"); let lease = scheduler @@ -2276,6 +2322,7 @@ mod tests { name: "multi-slot".to_string(), labels: HashMap::new(), capacity: WorkerCapacity { slots: 2 }, + identity: None, }) .expect("worker should register"); @@ -2328,6 +2375,7 @@ mod tests { name: "single-slot".to_string(), labels: HashMap::new(), capacity: WorkerCapacity { slots: 1 }, + identity: None, }) .expect("worker should register"); @@ -2412,6 +2460,7 @@ mod tests { ("pool".to_string(), "lighting".to_string()), ]), capacity: WorkerCapacity::default(), + identity: None, }) .expect("worker should register"); @@ -2449,6 +2498,7 @@ mod tests { name: "plain-worker".to_string(), labels: HashMap::new(), capacity: WorkerCapacity::default(), + identity: None, }) .expect("worker should register"); @@ -2494,6 +2544,7 @@ mod tests { name: "small-linux".to_string(), labels: HashMap::from([("ATTR.WORKER.OS.FAMILY".to_string(), "linux".to_string())]), capacity: WorkerCapacity { slots: 1 }, + identity: None, }) .expect("worker should register"); assert!(scheduler @@ -2506,6 +2557,7 @@ mod tests { name: "large-linux".to_string(), labels: HashMap::from([("attr.worker.os.family".to_string(), "linux".to_string())]), capacity: WorkerCapacity { slots: 2 }, + identity: None, }) .expect("worker should register"); let lease = scheduler @@ -2541,6 +2593,7 @@ mod tests { name: "local".to_string(), labels: HashMap::new(), capacity: WorkerCapacity::default(), + identity: None, }) .expect("worker should register"); @@ -2615,6 +2668,7 @@ mod tests { name: "local".to_string(), labels: HashMap::new(), capacity: WorkerCapacity::default(), + identity: None, }) .expect("worker should register"); let lease = scheduler.lease_task(worker.id).unwrap().unwrap(); @@ -2767,6 +2821,7 @@ mod tests { name: "local".to_string(), labels: HashMap::new(), capacity: WorkerCapacity::default(), + identity: None, }) .expect("worker should register") } diff --git a/crates/farm-worker/src/main.rs b/crates/farm-worker/src/main.rs index 603401c..f92cff0 100644 --- a/crates/farm-worker/src/main.rs +++ b/crates/farm-worker/src/main.rs @@ -189,6 +189,7 @@ async fn register_worker( capacity: WorkerCapacity { slots: args.slots.max(1), }, + identity: None, }) .send() .await? diff --git a/dashboard/src/types.ts b/dashboard/src/types.ts index 6aac1ea..4d1daaf 100644 --- a/dashboard/src/types.ts +++ b/dashboard/src/types.ts @@ -96,11 +96,19 @@ export interface ApiWorker { name: string; labels: Record; capacity: { slots: number }; + identity?: ApiWorkerIdentity | null; state: "online" | "offline"; registered_at: string; last_seen_at: string; } +export interface ApiWorkerIdentity { + provider: string; + subject: string; + attributes?: Record; + expires_at?: string | null; +} + export interface FarmLog { id: string; timestamp: string; diff --git a/docs/architecture.md b/docs/architecture.md index aa393ec..6f8fd96 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -40,6 +40,11 @@ Core endpoints: - `GET /v1/tasks/{task_id}/attempts/{attempt_id}/logs`: read logs for one durable attempt record. - `GET /v1/tasks/{task_id}/artifacts/{artifact_index}`: download a captured task artifact. +Backend, artifact, identity, and scheduler extension boundaries are defined in +[extension-contracts.md](extension-contracts.md). That note separates stable +client contracts from experimental backend and cloud-provider implementation +points. + Task leases have a configurable controller-side TTL. Workers renew leases while direct command and OpenJD tasks are still executing, so long-running work is not dispatched twice while its worker remains healthy. If a worker disappears and no @@ -57,6 +62,11 @@ worker. OpenJD `hostRequirements` are carried through the same scheduler path: standard amount capabilities are read from worker labels or slot capacity, and standard attribute capabilities are read from worker labels. +Worker registration accepts optional identity metadata for provisioned or +short-lived workers. Labels continue to describe scheduling capabilities; +identity remains provenance and authorization context so the scheduler can keep +task placement independent from cloud providers. + Lifecycle actions are idempotent where repeating the same request is safe, and invalid state transitions return a conflict response with a descriptive error. diff --git a/docs/extension-contracts.md b/docs/extension-contracts.md new file mode 100644 index 0000000..cd7e756 --- /dev/null +++ b/docs/extension-contracts.md @@ -0,0 +1,121 @@ +# Renderacre extension contracts + +This note defines the contracts that let Renderacre grow from a local farm into +hybrid or cloud-backed deployments without changing submitter, worker, or +dashboard clients for every backend swap. + +## Contract status + +Stable extension points: + +- The REST and Python job submission payloads, including OpenJD task conversion, + task requirements, shared limits, artifact metadata, attempt records, and log + references. +- Worker registration, heartbeat, lease, renewal, started, completion, and log + batch payloads. Workers that do not send optional fields remain valid. +- The optional worker identity object shape: `provider`, `subject`, + `attributes`, and `expires_at`. +- Scheduler decisions based on jobs, tasks, workers, leases, requirements, and + resource limits rather than a concrete database or cloud provider. +- Health, readiness, metrics, audit, dashboard, log, and artifact read models as + controller-facing operational contracts. + +Experimental extension points: + +- Postgres and managed control-plane storage implementations. +- Object storage log and artifact byte serving. +- Provider-specific worker identity validation, attestation, and credential + refresh. +- Multi-controller scheduling coordination, leader election, and cloud autoscale + policy hooks. + +Experimental implementations must preserve stable payloads unless a documented +versioned contract replaces them. + +## Storage backend responsibilities + +Every storage backend owns durable scheduler state, not scheduling policy. It +must persist jobs, tasks, attempt records, worker registrations, leases, shared +limits, farm logs, audit events, and observability counters using the existing +domain models. It must restore enough state after restart for expired lease +recovery, retry accounting, dashboard history, and audit review. + +The in-memory backend is for tests and demos only. It is allowed to lose all +state on process exit and should stay fast and dependency-free. + +The SQLite backend is the lightweight durable profile. It stores the scheduler +snapshot locally, writes changes atomically, and keeps REST workers, dashboard +reads, and Python submitters on the same public contract used by the in-memory +backend. + +A Postgres backend should provide the same scheduler state with transactional +writes, migrations, concurrent controller safety, and row-level or advisory +locking where needed. It must not add Postgres-specific fields to submitter or +worker payloads. + +A managed or cloud backend should expose the same controller contract while +moving persistence, backups, and operational scaling behind the backend +boundary. Provider-specific metadata belongs in backend configuration or audit +events, not in core task scheduling decisions. + +## Log and artifact storage responsibilities + +The scheduler owns attempt metadata: attempt id, attempt number, worker id, +state, timing, exit code, failure summary, stdout/stderr tails, artifact list, +and `log_ref`. A log or artifact storage backend owns byte persistence and +retrieval behind those references. + +The filesystem implementation records local artifact paths and serves readable +files through the controller. It must validate paths, avoid leaking unrelated +files, and preserve the `TaskArtifact` metadata returned by task completion. + +An object storage implementation should store logs and artifacts under immutable +attempt-scoped keys, return controller-resolvable references or signed reads, +and preserve the same artifact metadata. It should not decide which worker gets +which task, which retry is allowed, or which resource limit is consumed. + +## Worker identity and registration + +Workers may register with an optional identity: + +```json +{ + "name": "burst-worker-17", + "labels": { "pool": "burst", "app": "blender" }, + "capacity": { "slots": 8 }, + "identity": { + "provider": "aws-sts", + "subject": "i-0123456789abcdef0", + "attributes": { "region": "us-west-2" }, + "expires_at": "2026-05-28T10:30:00Z" + } +} +``` + +Labels remain scheduling capabilities. Identity is provenance and authorization +context for provisioned or short-lived workers. Current scheduler decisions do +not inspect `identity`; future auth layers can reject expired or invalid +identities before registration, and future reconciliation can mark workers +offline when an identity expires without a fresh registration or heartbeat. + +## Scheduler independence + +The scheduler consumes normalized domain state: pending work, worker capacity, +worker labels, OpenJD host requirements, shared resource limits, leases, and +attempt history. It must not branch on storage engine, object store, region, +cloud account, or provider-specific identity fields. + +Backends may decide how to persist, lock, replicate, and expose the state. They +must not change which task is runnable, which worker satisfies a requirement, or +how retries, leases, limits, pause, resume, cancel, and requeue transitions work. + +## Extension rules + +- Add optional fields with serde defaults when extending existing payloads. +- Keep provider-specific configuration out of stable task and worker contracts. +- Document any new stable field, endpoint, or state transition in the same PR + that introduces it. +- Add focused tests for contract preservation across in-memory and durable + backends when the change affects scheduler state. +- Prefer backend-specific adapters at the boundary over conditionals inside core + scheduling rules.