From 102498acec1b27c8179968b6cbc80cd76e0df3cb Mon Sep 17 00:00:00 2001 From: ABCxFF <79597906+abcxff@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:43:25 -0400 Subject: [PATCH] chore(container-runner): tighten comments --- container-runner/README.md | 6 +- .../e2e-test/configure-serverless.mjs | 2 +- .../examples/e2e-test/docker-compose.yml | 2 +- .../e2e-test/host/run-host-loadtest.sh | 2 +- .../examples/test-server/Dockerfile | 4 +- .../examples/test-server/server.mjs | 2 +- container-runner/src/actor.rs | 95 ++++------ container-runner/src/child.rs | 23 +-- container-runner/src/input.rs | 17 +- container-runner/src/main.rs | 162 +++++++----------- container-runner/src/monitor.rs | 35 +--- container-runner/src/proxy.rs | 12 +- 12 files changed, 135 insertions(+), 227 deletions(-) diff --git a/container-runner/README.md b/container-runner/README.md index 0a3fda9a8c..377c46ef2a 100644 --- a/container-runner/README.md +++ b/container-runner/README.md @@ -26,8 +26,8 @@ There are two working local paths: 2. Run the built Unity server behind `container-runner`, create a Rivet actor locally, and connect a FishNet client through the local Rivet guard URL. -A production image for **Rivet Compute** (the Cloud Run serverless model) is provided — -see [Rivet Compute](#rivet-compute) below. +A production image for **Rivet Compute** (the serverless model) is provided. See +[Rivet Compute](#rivet-compute) below. ## Project Layout @@ -207,7 +207,7 @@ Knobs: `LOAD_COUNT` (default 25), `LOAD_CONCURRENCY` (default 64). **Running the full 1000:** 1000 local instances is ~2000 processes (a Rust runner + Node child each) and needs a beefy host plus a raised `ulimit -n`. For a true 1000-container run, -point `load-test.mjs` at **Rivet Cloud** instead — Cloud Run scales the containers, no local +point `load-test.mjs` at **Rivet Cloud** instead, which scales the containers with no local limit. Set the engine env and let a single pool auto-scale: ```bash diff --git a/container-runner/examples/e2e-test/configure-serverless.mjs b/container-runner/examples/e2e-test/configure-serverless.mjs index cbec506474..a03ee0a11c 100644 --- a/container-runner/examples/e2e-test/configure-serverless.mjs +++ b/container-runner/examples/e2e-test/configure-serverless.mjs @@ -11,7 +11,7 @@ const body = { // Seconds the engine holds the /start request before draining to a fresh one. request_lifespan: 900, drain_grace_period: 30, - // 1:1 actor<->container mapping (Cloud Run concurrency=1 model). + // 1:1 actor<->container mapping (serverless concurrency=1 model). slots_per_runner: 1, max_runners: 1, max_concurrent_actors: 1, diff --git a/container-runner/examples/e2e-test/docker-compose.yml b/container-runner/examples/e2e-test/docker-compose.yml index c2aaae0f32..74c198107f 100644 --- a/container-runner/examples/e2e-test/docker-compose.yml +++ b/container-runner/examples/e2e-test/docker-compose.yml @@ -1,5 +1,5 @@ # Local end-to-end: self-hosted Rivet engine + the game container (container-runner -# wrapping the Node test server). All x86_64/amd64 to mirror Cloud Run. +# wrapping the Node test server). All x86_64/amd64 to mirror the serverless platform. # # Flow: # create actor (POST /actors) -> engine POSTs /api/rivet/start to game:8080 diff --git a/container-runner/examples/e2e-test/host/run-host-loadtest.sh b/container-runner/examples/e2e-test/host/run-host-loadtest.sh index 8133ca1276..2cde1503cf 100755 --- a/container-runner/examples/e2e-test/host/run-host-loadtest.sh +++ b/container-runner/examples/e2e-test/host/run-host-loadtest.sh @@ -4,7 +4,7 @@ # server), then drive a WebSocket ping-pong through the guard to every one. # # Each container-runner instance is one "container": its own front-door port, its own child -# port, and its own serverless runner pool (load-). This mirrors the Cloud Run 1:1 +# port, and its own serverless runner pool (load-). This mirrors the serverless 1:1 # actor<->container model locally, so `LOAD_COUNT` instances == that many containers. # # engine (:7420 guard, :7421 api) diff --git a/container-runner/examples/test-server/Dockerfile b/container-runner/examples/test-server/Dockerfile index 7274aa4739..24c54e509e 100644 --- a/container-runner/examples/test-server/Dockerfile +++ b/container-runner/examples/test-server/Dockerfile @@ -6,7 +6,7 @@ # Build from the RIVET REPO ROOT so the workspace and examples are in context. # Arch-agnostic: builds for the host/requested platform. # Local (native, e.g. arm64 Mac): docker build -f container-runner/examples/test-server/Dockerfile -t game:latest . -# Cloud Run (x86_64): docker build --platform linux/amd64 -f container-runner/examples/test-server/Dockerfile -t game:amd64 . +# Serverless (x86_64): docker build --platform linux/amd64 -f container-runner/examples/test-server/Dockerfile -t game:amd64 . # # Production pattern (per spec) — instead of building from source, curl a released # binary into an existing image: @@ -33,7 +33,7 @@ COPY --from=builder /usr/local/bin/rivet-container-runner /usr/local/bin/rivet-c COPY container-runner/examples/test-server/ /app/test-server/ RUN cd /app/test-server && npm install --omit=dev -# Cloud Run sets $PORT (the serverless front door). The child listens on CHILD_PORT. +# The serverless platform sets $PORT (the front door). The child listens on CHILD_PORT. ENV PORT=8080 \ CHILD_PORT=7770 \ RIVET_ACTOR_NAME=game diff --git a/container-runner/examples/test-server/server.mjs b/container-runner/examples/test-server/server.mjs index 2822669542..5c81a47d35 100644 --- a/container-runner/examples/test-server/server.mjs +++ b/container-runner/examples/test-server/server.mjs @@ -2,7 +2,7 @@ // // This is the child process that container-runner (rivet-container-runner) spawns inside // the container. It stands in for a real Unity FishNet dedicated server while we validate -// the Rivet -> Cloud Run -> container pipeline. +// the Rivet -> serverless -> container pipeline. // // Normal behavior: // - Binds HTTP+WebSocket on $PORT (default 7770) on 0.0.0.0. diff --git a/container-runner/src/actor.rs b/container-runner/src/actor.rs index 7a8d3cea2d..86b2befbc3 100644 --- a/container-runner/src/actor.rs +++ b/container-runner/src/actor.rs @@ -1,11 +1,8 @@ //! The `GameServer` actor: wraps one child game-server process per actor. //! -//! Lifecycle: `on_start` reserves a port and spawns the child, waiting for -//! readiness (so the actor is never reported ready before the child listens), -//! `run` is a watchdog that reports unexpected child exits, -//! `on_fetch`/`on_websocket` proxy tunneled traffic to the child's port, and -//! `on_destroy` stops the child while the instance stays warm for the next -//! placement. +//! `on_start` reserves a port and spawns the child (waiting for readiness), `run` +//! watchdogs unexpected child exits, `on_fetch`/`on_websocket` proxy tunneled +//! traffic to the child, and `on_sleep`/`on_destroy` stop it. use std::sync::{Arc, LazyLock}; @@ -21,9 +18,8 @@ use crate::{ request_exit, reserve_child_port, runner_config, }; -/// Live actor contexts on this instance, keyed by actor id. Lets the process -/// shutdown path report actors as crashed when the platform reclaims the -/// container out from under them. +/// Live actor contexts keyed by actor id, so the shutdown path can report actors +/// as crashed when the platform reclaims the container. static ACTOR_CTXS: LazyLock>> = LazyLock::new(scc::HashMap::new); @@ -32,14 +28,12 @@ pub struct GameServer { } impl GameServer { - /// Shared teardown for sleep and destroy: for a game server the two are - /// materially the same event, because the in-memory match state lives in - /// the child and cannot outlive the container. The launch spec is the - /// persisted actor state, so a later wake respawns an equivalent child. + /// Shared teardown for sleep and destroy. For a game server they are the same + /// event: match state lives in the child and cannot outlive the container, and + /// a later wake respawns an equivalent child from the persisted launch spec. async fn stop_child(&self, actor_id: &str, reason: &str) { - // Remove from the registry FIRST so the watchdog treats the exit as - // deliberate, then stop. `stop` is idempotent if the process shutdown - // sweep already stopped this child. + // Remove from the registry first so the watchdog treats the exit as + // deliberate. `stop` is idempotent if the shutdown sweep already ran. children().remove_async(actor_id).await; ACTOR_CTXS.remove_async(actor_id).await; let child = self.child.lock().await.take(); @@ -48,11 +42,9 @@ impl GameServer { release_child_port(child.child_port).await; } - // Exit the whole process once the last child on this instance stops. The - // runner is PID 1, so `request_exit` cancels `EXIT`, which wakes `main` - // to run the graceful envoy close and then return, stopping the container - // so the platform reaps it. Guarded on an empty registry so a multi-actor - // instance does not tear down siblings still hosting a child. + // Exit the process once the last child stops: `request_exit` cancels + // `EXIT`, waking `main` to close the envoy and return (the runner is + // PID 1). Guarded on an empty registry so siblings survive. if children().is_empty() { request_exit(actor_id, reason); } else { @@ -64,12 +56,9 @@ impl GameServer { } } - /// Engine pause path (sleep, lost, going-away). Give the child up to - /// `DRAIN_GRACE` to finish its in-flight work and exit on its own before we - /// force a stop; the child is not signalled during the window. A natural - /// child exit ends the wait immediately, and a platform SIGTERM (which - /// cancels the exit token) cuts it short so the reclaim's SIGTERM→SIGKILL - /// budget is honored. + /// Engine pause (sleep, lost, going-away): let the child finish and exit on its + /// own for up to `DRAIN_GRACE` before forcing a stop. A child exit or a platform + /// SIGTERM (which cancels the exit token) ends the wait early. async fn drain_then_stop_child(&self, actor_id: &str, reason: &str) { let child = self.child.lock().await.clone(); if let Some(child) = child { @@ -118,10 +107,8 @@ impl Actor for GameServer { let actor_id = ctx.actor_id().to_string(); let key = actor_key_string(&ctx); - // Surface the resource monitor's status here, tagged with the actor id, so - // it is visible in actor-scoped log views. The monitor's own enable/disable - // logs are process-level and have no actor id, so they are filtered out of - // those views. + // Tagged with the actor id so it shows in actor-scoped log views; the + // monitor's own process-level enable/disable logs are filtered out there. tracing::info!( actor_id = %actor_id, resource_monitor_enabled = crate::monitor::enabled(), @@ -129,9 +116,8 @@ impl Actor for GameServer { "resource monitor status" ); - // An engine retry for an actor that is already running here must be an - // idempotent no-op: rejecting it would make the engine tear down a - // healthy actor. + // An engine retry for an already-running actor must be an idempotent no-op; + // rejecting it would make the engine tear down a healthy actor. if let Some(existing) = children().read_async(&actor_id, |_, c| c.clone()).await { if !existing.has_exited() { println!( @@ -172,9 +158,8 @@ impl Actor for GameServer { key: key.clone(), }; - // Version line, tagged with the actor id so it is visible in actor-scoped - // logs. `git_sha` is omitted entirely when unknown rather than logged as - // "unknown". + // Tagged with the actor id for actor-scoped logs; `git_sha` is omitted when + // unknown rather than logged as "unknown". match crate::git_sha() { Some(git_sha) => tracing::info!( actor_id = %actor_id, @@ -200,17 +185,13 @@ impl Actor for GameServer { Ok(child) => Arc::new(child), Err(err) => { release_child_port(child_port).await; - // A failed start is this actor's alone and does not take the - // instance down. The container stays warm and ready for the next - // placement, and stays alive long enough for the log agent to - // drain the failure logs before the platform reaps it. + // A failed start is this actor's alone; it does not take down others. return Err(err); } }; - // The global registry lets the process shutdown path stop children - // even when actor hooks never run, and arbitrates the deliberate-stop - // vs unexpected-exit race for the watchdog in `run`. + // The global registry lets the shutdown path stop children when hooks never + // run, and arbitrates the deliberate-stop vs unexpected-exit race in `run`. if children() .insert_async(actor_id.clone(), child.clone()) .await @@ -221,19 +202,16 @@ impl Actor for GameServer { release_child_port(child_port).await; anyhow::bail!("a child for actor {actor_id} is already registered"); } - // Register only now that startup has succeeded. Registering earlier would - // leak an entry for any generation whose start failed, since a failed - // start never runs on_destroy/on_sleep to remove it. + // Register only after startup succeeds; a failed start never runs a stop + // hook to remove the entry, so registering earlier would leak it. register_ctx(&actor_id, &ctx).await; *self.child.lock().await = Some(child); Ok(()) } - /// Watchdog: waits for the child to exit. Deliberate stops remove the - /// child from the global registry first, so winning the `remove` race - /// means the exit was unexpected and the actor must be torn down. A clean - /// exit (code 0) destroys the actor; any other exit returns an error so - /// the framework reports an errored stop and the engine records the crash. + /// Watchdog for the child exiting. Deliberate stops remove it from the registry + /// first, so winning the `remove` race means the exit was unexpected: a clean + /// exit destroys the actor, any other reports an errored stop (a crash). async fn run(self: Arc, ctx: Ctx) -> Result<()> { let Some(child) = self.child.lock().await.clone() else { anyhow::bail!("run: child process was never spawned"); @@ -294,10 +272,8 @@ impl Actor for GameServer { crate::proxy::ws_proxy(child_port, path, ws).await } - /// Engine-initiated sleep. `no_sleep` suppresses idle sleep, but the - /// engine can still sleep an actor (dashboard, crash policy, eviction - /// ahead of instance retirement); leaving the child running would orphan - /// it on an instance the engine considers vacated. + /// Engine-initiated sleep. `no_sleep` blocks only idle sleep; the engine can + /// still sleep an actor (dashboard, crash policy, eviction), so we stop the child. async fn on_sleep(self: Arc, ctx: Ctx) -> Result<()> { self.drain_then_stop_child(ctx.actor_id(), "actor sleeping").await; Ok(()) @@ -318,10 +294,9 @@ async fn register_ctx(actor_id: &str, ctx: &Ctx) { .await; } -/// Report every live actor on this instance as crashed. Called when the -/// platform reclaims the container (an unexpected SIGTERM) so the reclaim -/// surfaces as a crash on the engine instead of a silent reallocation. Runs -/// while the envoy is still connected so the crash reaches the engine. +/// Report every live actor as crashed. Called when the platform reclaims the +/// container (unexpected SIGTERM), while the envoy is still connected, so the +/// reclaim surfaces as a crash on the engine instead of a silent reallocation. pub async fn crash_all_actors(message: &str) { let mut ctxs = Vec::new(); ACTOR_CTXS diff --git a/container-runner/src/child.rs b/container-runner/src/child.rs index c9d5843ae6..907fc2ffbb 100644 --- a/container-runner/src/child.rs +++ b/container-runner/src/child.rs @@ -1,9 +1,8 @@ //! Child game-server process management: spawn, log piping, readiness, SIGTERM stop. //! -//! Ownership model: a dedicated "reaper" task exclusively owns the `tokio::process::Child` -//! and awaits its exit, publishing the result on a `watch` channel. `stop()` and readiness -//! checks signal/observe via the pid and the watch channel, so they never contend for the -//! child handle (which would deadlock against the reaper's long-lived `wait()`). +//! A dedicated reaper task owns the `tokio::process::Child` and publishes its exit on a +//! `watch` channel; `stop()` and readiness checks signal/observe via the pid and channel, +//! so they never contend for the child handle (which would deadlock the reaper's `wait()`). use std::collections::HashMap; use std::net::Ipv4Addr; @@ -72,12 +71,9 @@ impl ChildProcess { let prefix = log_prefix(&actor_id, key.as_deref()); - // Guarantee the child port is free BEFORE spawning. Otherwise a stale child from a - // prior start (in a reused container instance) still holding the port would make - // `wait_until_ready` below false-positive: it connects to the OLD listener and - // reports the NEW child "ready" even though the new child failed to bind - // (`Address already in use`) and is dead. Refuse the start with a clear diagnostic - // instead — this container hosts exactly one game server on a fixed port. + // Refuse to spawn if the port is already taken. A stale child from a prior start + // still holding it would make `wait_until_ready` false-positive on the OLD listener + // while the new child dies with `Address already in use`. if TcpStream::connect((Ipv4Addr::LOCALHOST, child_port)) .await .is_ok() @@ -85,7 +81,7 @@ impl ChildProcess { anyhow::bail!( "child port {child_port} is already in use before spawning `{program}`: a \ previous game server is still running in this container. container-runner \ - hosts one actor per container — configure the serverless runner with \ + hosts one actor per container; configure the serverless runner with \ max_concurrent_actors=1 and platform request concurrency=1." ); } @@ -143,9 +139,8 @@ impl ChildProcess { exited_rx, }; - // If the child crashes before opening its port, or never opens it, make sure we - // don't leave it running: kill it before surfacing the start failure. (The reaper - // task owns the tokio Child, so dropping `this` alone would NOT kill a hung child.) + // Kill the child before surfacing a readiness failure; dropping `this` alone would + // not (the reaper owns the tokio Child), leaving a hung child running. if let Err(err) = this.wait_until_ready(readiness_timeout).await { this.stop(Duration::from_secs(2)).await; return Err(err); diff --git a/container-runner/src/input.rs b/container-runner/src/input.rs index 1b39d3c089..c39fb6d0bd 100644 --- a/container-runner/src/input.rs +++ b/container-runner/src/input.rs @@ -1,19 +1,16 @@ //! The actor input payload describing how to launch the child game server. //! -//! Everything the game server needs to launch (command, args, env, port) is -//! carried in the actor's create-time `input` payload, CBOR-encoded per the -//! RivetKit convention. All fields are optional; anything omitted falls back -//! to the CLI-provided template (`rivet-container-runner -- `). -//! The decoded input is also the actor's persisted state so a woken actor -//! restores the same launch spec without re-decoding input. +//! The command, args, env, and port are carried in the actor's create-time `input` +//! (CBOR per RivetKit); anything omitted falls back to the CLI template +//! (`rivet-container-runner -- `). This is also the actor's persisted +//! state, so a woken actor restores the same launch spec. use serde::{Deserialize, Serialize}; use std::collections::HashMap; -/// Shape of the actor `input` payload. Unknown fields are ignored rather than -/// rejected: this type is also the persisted actor state, and a strict decode -/// would break waking actors after a rollback to a binary that predates a -/// newly added field. +/// Shape of the actor `input` payload. Unknown fields are ignored, not rejected: +/// this is also the persisted state, and a strict decode would break waking actors +/// after a rollback to a binary predating a new field. #[derive(Debug, Default, Serialize, Deserialize)] pub struct ActorInput { /// Overrides the CLI command template entirely (program + fixed args). diff --git a/container-runner/src/main.rs b/container-runner/src/main.rs index 5c2bf9bd5f..cf0ebb854a 100644 --- a/container-runner/src/main.rs +++ b/container-runner/src/main.rs @@ -1,20 +1,14 @@ -//! `rivet-container-runner` — a RivetKit serverless app that hosts a single -//! actor by spawning a child game-server process and proxying Rivet's -//! tunneled traffic to it. +//! `rivet-container-runner`: a RivetKit serverless app that hosts actors by +//! spawning one child game-server process per actor and proxying Rivet's tunneled +//! traffic to it. //! -//! Serverless model: the engine's `POST /api/rivet/start` starts this -//! container (served by rivetkit-core's serverless runtime). On actor start -//! the `GameServer` actor spawns the child (`-- `), pipes its -//! logs to stdout prefixed with the actor id + key, and proxies inbound -//! HTTP/WebSocket (arriving over Rivet's tunnel) to the child's local port. -//! On actor stop it SIGTERMs the child. -//! -//! The runner hosts as many concurrent actors as the engine places on it, -//! each with its own child process on its own port; the pool's request -//! concurrency decides how many that is (1 in the recommended game-server -//! setup). The instance stays warm after its last actor stops and never -//! self-exits; the engine reaps it by draining the `/start` connection once -//! the request lifespan elapses, or the platform sends a SIGTERM. +//! The engine's `POST /api/rivet/start` boots this container. On actor start the +//! `GameServer` actor spawns the child (`-- `), pipes its logs to +//! stdout prefixed with the actor id + key, and proxies inbound HTTP/WebSocket to +//! the child's local port. On actor stop it SIGTERMs the child; once the last +//! child stops the process exits so the platform reaps the instance. The engine +//! decides how many actors land here, each on its own port (1 in the recommended +//! game-server setup). mod actor; mod child; @@ -46,9 +40,9 @@ pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION"); /// image build context excludes `.git`). pub(crate) const GIT_SHA: &str = env!("CONTAINER_RUNNER_GIT_SHA"); -/// The effective git SHA, or `None` when unknown. Prefers the build-time SHA and -/// falls back to a runtime `OVERRIDE_GIT_SHA` env var, so a deploy that cannot -/// inject a build arg can still surface it via an environment variable. +/// The effective git SHA, or `None` when unknown. Prefers the build-time SHA, +/// falling back to the `OVERRIDE_GIT_SHA` env var for deploys that cannot inject a +/// build arg. pub(crate) fn git_sha() -> Option { if GIT_SHA != "unknown" { return Some(GIT_SHA.to_string()); @@ -73,39 +67,31 @@ pub struct RunnerConfig { // configuration is ambient process state set once in `main`. static RUNNER_CONFIG: OnceLock> = OnceLock::new(); -/// Running children keyed by actor id. Owned globally (not only by actors) so -/// the process shutdown path can stop children even if actor hooks never run, -/// and so the watchdog and stop paths can arbitrate who reports an exit. +/// Running children keyed by actor id. Global (not per-actor) so the shutdown path +/// can stop children even when hooks never run, and the watchdog can arbitrate exits. static CHILDREN: LazyLock>> = LazyLock::new(scc::HashMap::new); -/// Child ports currently reserved by a spawning or running child. Multiple -/// actors may run concurrently (engine placement decides how many land here), -/// so each child needs its own port and concurrent starts must not race to -/// the same one. +/// Ports reserved by spawning or running children. Multiple actors can run here at +/// once, so each needs its own port and concurrent starts must not race for one. static RESERVED_PORTS: LazyLock> = LazyLock::new(scc::HashSet::new); -/// Cancelled to bring the whole process down (actor stopped, failed start, or -/// signal). `main` owns the exit sequencing. +/// Cancelled to bring the whole process down (last child stopped or a signal). +/// `main` owns the exit sequencing. static EXIT: LazyLock = LazyLock::new(CancellationToken::new); -/// Set when the process is shutting down because the PLATFORM sent a signal. -/// The hosting platform gives a container only a bounded window (often ~10 -/// seconds) between SIGTERM and SIGKILL, so every grace period on this path must -/// fit that budget; engine-initiated stops keep the full configured grace -/// (their budget is the pool's drain grace period instead). +/// Set when a platform signal is driving shutdown. The platform gives only a +/// bounded window (~10s) between SIGTERM and SIGKILL, so grace periods on this +/// path must fit that budget. static SIGNAL_SHUTDOWN: AtomicBool = AtomicBool::new(false); -/// Set when shutdown was triggered by a platform SIGTERM (an instance reclaim), -/// as opposed to a local SIGINT (developer Ctrl-C). Distinguishes a reclaim -/// from a manual stop for downstream shutdown handling. +/// Set when shutdown was a platform SIGTERM (instance reclaim) rather than a local +/// SIGINT (Ctrl-C), so the reclaim can be reported as an actor crash. static PLATFORM_RECLAIM: AtomicBool = AtomicBool::new(false); -/// How long the platform gives this container between SIGTERM and SIGKILL. -/// Defaults to 9s, one second under the common ~10s platform budget so the whole -/// teardown lands before SIGKILL; keep it in sync with the platform's actual -/// budget via RIVET_SIGTERM_BUDGET_SECS. On the signal path the engine drain and -/// the child kills run concurrently, each bounded by this full budget. +/// SIGTERM→SIGKILL window the platform gives this container. Defaults to 9s, just +/// under the common ~10s budget; override with RIVET_SIGTERM_BUDGET_SECS. On the +/// signal path the engine drain and the child kills share this budget concurrently. static SIGTERM_BUDGET: LazyLock = LazyLock::new(|| { let secs = std::env::var("RIVET_SIGTERM_BUDGET_SECS") .ok() @@ -115,11 +101,9 @@ static SIGTERM_BUDGET: LazyLock = LazyLock::new(|| { Duration::from_secs(secs) }); -/// How long an engine-initiated pause (sleep, lost, going-away) lets the child -/// keep serving and finish its own work before we force a stop. The child is not -/// signalled during this window; it either exits on its own or is SIGTERM'd at -/// the end. Defaults to 15 minutes. It must fit inside the engine's per-runner -/// `drain_grace_period` or a platform reclaim cuts it short. +/// How long an engine pause (sleep, lost, going-away) lets the child keep serving +/// and exit on its own before we SIGTERM it. Defaults to 15 min (RIVET_DRAIN_GRACE_SECS); +/// must fit inside the engine's per-runner `drain_grace_period` or a reclaim cuts it short. static DRAIN_GRACE: LazyLock = LazyLock::new(|| { let secs = std::env::var("RIVET_DRAIN_GRACE_SECS") .ok() @@ -133,8 +117,8 @@ pub fn drain_grace() -> Duration { *DRAIN_GRACE } -/// The cancellation token that fires on a platform shutdown signal. Used to cut -/// a drain wait short so the platform's SIGTERM→SIGKILL budget is honored. +/// Token that fires on a platform shutdown signal. Cuts a drain wait short so the +/// platform's SIGTERM→SIGKILL budget is honored. pub fn exit_token() -> &'static CancellationToken { &EXIT } @@ -164,11 +148,9 @@ pub async fn active_actor_ids() -> Vec { ids } -/// Reserve a local port for a new child. An explicit `input.port` is honored -/// or refused if another child holds it; otherwise the first free port at or -/// above the CLI default is picked. The reservation guards the window between -/// port selection and the child actually binding; release it via -/// [`release_child_port`] once the child is gone. +/// Reserve a local port for a new child. An explicit `input.port` is honored (or +/// refused if held); otherwise the first free port at or above the default is used. +/// Guards selection-to-bind; release via [`release_child_port`] once the child is gone. pub async fn reserve_child_port(preferred: Option, default: u16) -> Result { if let Some(port) = preferred { if RESERVED_PORTS.insert_async(port).await.is_err() { @@ -210,31 +192,28 @@ pub async fn release_child_port(port: u16) { } /// The SIGTERM→SIGKILL window for the child: always `SIGTERM_BUDGET`, whatever -/// triggered the stop, so the child never gets a different kill deadline on one -/// path than another. The wait *before* SIGTERM (letting the child exit on its -/// own during an engine pause) is separate; see [`drain_grace`]. +/// triggered the stop. The wait before SIGTERM (an engine pause) is separate; see +/// [`drain_grace`]. pub fn effective_stop_grace() -> Duration { *SIGTERM_BUDGET } -/// End the process. Driven by a platform shutdown signal or by the last child on -/// this instance stopping (see `stop_child`). The runner is PID 1 in the image, -/// so cancelling `EXIT` wakes `main` to run the graceful envoy close and return, -/// which stops the container and lets the platform reap the instance. +/// End the process. Driven by a platform signal or by the last child stopping (see +/// `stop_child`). The runner is PID 1, so cancelling `EXIT` wakes `main` to run the +/// graceful envoy close and return, which stops the container. pub fn request_exit(actor_id: &str, reason: &str) { tracing::info!(actor_id = %actor_id, reason, "shutting down container"); EXIT.cancel(); } -/// Stable per-process identifier, generated once. The runner is PID 1 in the -/// image, so one boot id == one container instance. Logged at startup and on -/// every actor start so an actor can be attributed to an instance (the log -/// stream carries no instance id). +/// Stable per-process id, generated once. The runner is PID 1, so one boot id == +/// one container instance. Logged per actor start so an actor can be attributed to +/// an instance (the log stream carries no instance id). pub fn boot_id() -> &'static str { static BOOT_ID: OnceLock = OnceLock::new(); BOOT_ID.get_or_init(|| { - // 9 random bytes -> 12 chars. Falls back to a fixed marker if the - // CSPRNG is unavailable, which would itself be worth seeing in logs. + // 9 random bytes -> 12 chars. Falls back to a marker if the CSPRNG is + // unavailable, which is itself worth seeing in logs. let mut buf = [0u8; 9]; match std::fs::File::open("/dev/urandom").and_then(|mut f| f.read_exact(&mut buf)) { Ok(()) => base64url_nopad(&buf), @@ -272,8 +251,7 @@ fn base64url_nopad(input: &[u8]) -> String { long_about = None, )] struct Args { - /// Serverless HTTP front-door port. Rivet Compute injects RIVET_PORT (other - /// serverless platforms use the conventional PORT); resolved in `main` as + /// Serverless HTTP front-door port. Resolved in `main` as /// --port > RIVET_PORT > PORT > 8080. #[arg(long)] port: Option, @@ -332,7 +310,7 @@ async fn async_main() -> Result<()> { let boot_id = boot_id(); tracing::info!(?args, %boot_id, "starting container-runner"); - // Front-door port: Rivet Compute injects RIVET_PORT; other serverless platforms use the conventional PORT. + // Front-door port: Rivet Compute injects RIVET_PORT; other platforms use PORT. let port = args .port .or_else(|| env_u16("RIVET_PORT")) @@ -353,11 +331,9 @@ async fn async_main() -> Result<()> { ActorConfig { // Game servers hold live in-memory state; never idle-sleep the actor. no_sleep: true, - // Core force-aborts the stop hook after this deadline. `on_sleep` - // legitimately runs for the whole drain window plus the SIGTERM - // budget, so the deadline must outlast both or core would abort the - // drain mid-flight and leak the child. The extra 5s keeps the - // deadline from racing a hook that finishes right on time. + // Core force-aborts the stop hook at this deadline, so it must outlast the + // full drain window plus the SIGTERM budget (with a small margin) or the + // drain is cut short and the child leaks. sleep_grace_period: *DRAIN_GRACE + *SIGTERM_BUDGET + Duration::from_secs(5), sleep_grace_period_overridden: true, ..Default::default() @@ -394,27 +370,15 @@ async fn async_main() -> Result<()> { )); tracing::info!(port, "container-runner serverless front door listening"); - // Wait for an exit request, then tear down. Two shapes depending on why: - // - // Signal (platform is reclaiming the instance): kill our children AND - // notify the engine at the SAME time, each bounded by the full SIGTERM - // budget. The direct sweep is what guarantees children die within budget - // rather than waiting on an engine round-trip to run our on_destroy hooks; - // notifying the engine in parallel just lets it start re-placing actors - // immediately. Bounding the drain means an unreachable engine cannot eat - // the budget the children need. - // - // Actor-driven exit (the last child stopped, so `stop_child` cancelled - // `EXIT`): no platform deadline. The child is already reaped by the hook - // that triggered the exit (the sweep is a no-op backstop), and the runtime - // drains unbounded so the /start SSE flushes cleanly. + // Wait for an exit request, then tear down. A platform reclaim (signal) kills + // children and notifies the engine at once, each bounded by the SIGTERM budget. + // An actor-driven exit (last child stopped) has no deadline: the child is already + // reaped, so the runtime drains unbounded. EXIT.cancelled().await; if SIGNAL_SHUTDOWN.load(Ordering::Acquire) { - // A platform SIGTERM reclaims this instance. Report every actor as crashed - // before draining so an unexpected SIGTERM (OOM or the ~60 minute request - // cap) surfaces as a crash on the engine instead of a silent reallocation. - // This runs while the envoy is still connected so the crash reaches the - // engine. A local SIGINT (Ctrl-C) drains gracefully without a crash. + // A platform SIGTERM reclaims this instance. Report actors as crashed while + // the envoy is still connected so the reclaim (OOM or the ~60 min request + // cap) surfaces as a crash, not a silent reallocation. SIGINT drains cleanly. if PLATFORM_RECLAIM.load(Ordering::Acquire) { crate::actor::crash_all_actors( "runner received unexpected platform SIGTERM, likely OOM or running longer than 60 minutes", @@ -446,11 +410,9 @@ async fn async_main() -> Result<()> { Ok(()) } -/// Stop every child still in the registry. Actor `on_destroy` normally reaps -/// its own child first; this is the belt-and-suspenders sweep for the signal -/// path so children are never orphaned. Children are stopped concurrently so -/// each gets the full `grace` within the SIGTERM budget instead of queueing -/// behind the others. +/// Stop every child still in the registry, concurrently so each gets the full +/// `grace`. Actor hooks normally reap their own child first; this is the sweep for +/// the signal path so children are never orphaned. async fn stop_all_children(grace: Duration) { let mut children: Vec> = Vec::new(); CHILDREN @@ -482,8 +444,8 @@ fn spawn_signal_handler() { tokio::select! { _ = sigterm.recv() => { PLATFORM_RECLAIM.store(true, Ordering::Release); - // Attribute the reclaim to each running actor so it is visible in - // actor-scoped logs, not only the process-level log stream. + // Attribute the reclaim to each running actor so it shows in + // actor-scoped logs, not only the process-level stream. let mut actor_ids = Vec::new(); CHILDREN .retain_async(|actor_id, _| { diff --git a/container-runner/src/monitor.rs b/container-runner/src/monitor.rs index 3f9afea9d9..a5cbc88598 100644 --- a/container-runner/src/monitor.rs +++ b/container-runner/src/monitor.rs @@ -1,33 +1,14 @@ //! Periodic instance resource monitor. //! -//! Opt-in via the [`ENABLE_ENV`] environment variable. When enabled it samples -//! memory and CPU usage every [`sample_interval`] and logs them, so memory -//! growth toward the limit (and the OOM that follows) is visible in the logs at -//! fine granularity. Disabled by default so nothing is logged unless explicitly -//! turned on. +//! Opt-in via [`ENABLE_ENV`]. When on, it samples memory and CPU every +//! [`sample_interval`] and logs them so growth toward the limit (and the OOM that +//! follows) is visible at fine granularity. //! -//! Memory and CPU are detected independently, because the gVisor sandbox -//! exposes cgroup v1 memory but no cgroup v2 or cgroup v1 CPU accounting, so the -//! two counters legitimately come from different sources. -//! -//! Only memory *usage* is reported, not a limit or percentage: under gVisor both -//! the cgroup `memory.limit_in_bytes` and `/proc/meminfo` report the sandbox size -//! rather than the container's configured limit, so any percentage would be -//! misleading. -//! -//! Memory sources, in preference order: -//! - **cgroup v2** `memory.current` (real Linux). Exact. -//! - **cgroup v1** `memory/memory.usage_in_bytes` (gVisor sandbox). -//! Container-wide usage (all processes plus page cache). -//! - **`/proc/meminfo`** last resort. Under gVisor this reflects the whole -//! sandbox, not the container, so it is only an approximation. -//! -//! CPU sources, in preference order: -//! - **cgroup v2** `cpu.stat`. -//! - **`/proc/stat`**, the gVisor sandbox fallback (sandbox-wide, approximate). -//! -//! If neither a memory nor a CPU source is readable the monitor logs once and -//! disables itself. +//! Memory and CPU are detected independently (see [`MemSource`]/[`CpuSource`]): the +//! gVisor sandbox exposes cgroup v1 memory but no cgroup v2 or v1 CPU, so the two +//! counters can come from different sources. Only memory *usage* is reported, not a +//! limit or percentage, since under gVisor the reported limit is the sandbox size, +//! not the container's. If neither counter is readable the monitor disables itself. use std::time::{Duration, Instant}; diff --git a/container-runner/src/proxy.rs b/container-runner/src/proxy.rs index 983b2aaedc..62d5c0461e 100644 --- a/container-runner/src/proxy.rs +++ b/container-runner/src/proxy.rs @@ -1,9 +1,8 @@ //! Bridges Rivet's decoded tunnel traffic to the child game server's local port. //! //! The rivetkit runtime reassembles tunnel frames into decoded `Request`s and -//! `WebSocket` streams (see `Actor::on_fetch` / `::on_websocket`). This module -//! forwards those to `127.0.0.1:` and back — the glue the runtime -//! deliberately leaves to the actor. +//! `WebSocket` streams (`Actor::on_fetch`/`::on_websocket`); this module forwards +//! them to `127.0.0.1:` and back, the glue the runtime leaves to the actor. use std::collections::HashMap; use std::sync::Arc; @@ -74,10 +73,9 @@ enum ClientEvent { Close(u16, String), } -/// Cap on client frames buffered toward a slow child. The old async pump -/// applied tunnel backpressure by awaiting the child sink; the sync message -/// callback cannot await, so a bounded queue plus a loud connection failure -/// on overflow replaces silent unbounded buffering. +/// Cap on client frames buffered toward a slow child. The sync message callback +/// cannot await for backpressure, so overflow fails the connection loudly instead +/// of buffering unbounded. const CLIENT_TO_CHILD_QUEUE: usize = 1024; /// Dial the child's WebSocket endpoint and pump frames in both directions: