Add opt-in exclusive model-container ownership

CHANGELOG.md

@@ -8,6 +8,41 @@ measurement is the useful part.
 `docs/LEARNED.md` carries the full reasoning; this file carries what
 changed. Each entry names the section to read for the numbers behind it.
 
+## 0.6.7 — 2026-08-10
+
+The container format, arithmetic, routing, and default open behavior are
+unchanged. A host can now ask cooperating WASTE processes for exclusive
+ownership of one container before model-sized allocation begins. This is a
+host-policy mechanism for a workstation daemon, not RAM accounting or a data
+lock: containers remain read-only, concurrent opens remain the default, and an
+unsupported advisory lock does not make a readable model unavailable.
+
+**Callers must recompile against this header.** `exclusive_open` was appended
+to `waste_cfg`, and `WASTE_E_BUSY` was added to `waste_status`. The library is
+still pre-1.0 and does not promise a stable ABI; `serve/engine.py`'s ctypes
+mirror moved with the C header.
+
+### Added
+
+- **Opt-in single-process container ownership**
+  ([#29](https://github.com/sqliteai/waste/pull/29)). On POSIX hosts,
+  `waste_cfg.exclusive_open`, or `--exclusive-open` in the CLI and server,
+  takes a non-blocking advisory `flock` on the container directory. Multiple
+  contexts in one process share a device/inode-keyed reference; a cooperating
+  process that also requests exclusivity receives `WASTE_E_BUSY`. The last
+  close and every planning, budget, and partial-load failure release ownership;
+  descriptors are close-on-exec, and a forked child discards the copied
+  registry. Windows keeps its existing lifecycle behavior.
+
+### Fixed
+
+- **Advisory locking fails open when ownership cannot be established.** Only
+  actual `EWOULDBLOCK`/`EAGAIN` contention returns `WASTE_E_BUSY`. A directory
+  that is search-only, a filesystem without `flock`, or another non-contention
+  locking failure continues through the ordinary model-open path. This keeps
+  external FUSE, SMB, and NFS containers usable and leaves their real read
+  errors to the existing loader diagnostics.
+
 ## 0.6.6 — 2026-08-05
 
 The engine decodes exactly as 0.6.5 did and no container format moved. Two

Makefile

@@ -228,7 +228,7 @@ waste$(EXE): cli/main.o libwaste.a
 # the two failures tests/run.sh was written to catch, so a binary that
 # `test` builds and `clean` forgets defeats the check meant to notice it.
 TESTNAMES := test_kda test_container test_forward test_tokenizer test_k3parts \
-             test_state test_vision test_image test_memory test_cpus sweep
+             test_state test_vision test_image test_memory test_cpus test_lock sweep
 TESTBINS  := $(addsuffix $(EXE),$(TESTNAMES))
 
 test: $(TESTBINS)
@@ -274,6 +274,9 @@ test_memory$(EXE): tests/test_memory.o src/memory.o
 test_cpus$(EXE): tests/test_cpus.o
 	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
 
+test_lock$(EXE): tests/test_lock.o libwaste.a
+	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
+
 %.o: %.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 

cli/main.c

@@ -112,7 +112,7 @@ typedef struct {
     uint64_t budget;
     uint32_t ctx, max_tokens;
     float temperature, top_p;
-    int top_k, threads, quiet, learn, json, no_echo;
+    int top_k, threads, quiet, learn, json, no_echo, exclusive_open;
     int media_inlined;              /* the media block is already in the
                                        prompt string, inside the user turn */
     uint64_t seed;
@@ -190,6 +190,7 @@ static int parse_opts(int argc, char **argv, int from, opts *o)
         else if (!strcmp(a, "--json")) o->json = 1;
         else if (!strcmp(a, "--raw")) o->raw = 1;
         else if (!strcmp(a, "--verify")) o->verify = 1;
+        else if (!strcmp(a, "--exclusive-open")) o->exclusive_open = 1;
         else if (!strcmp(a, "-")) {                /* explicit stdin */
             if (o->n_pos >= MAX_POS) { fprintf(stderr, "too many arguments\n"); return -1; }
             o->pos[o->n_pos++] = "-";
@@ -273,6 +274,7 @@ static waste_status open_model(const char *path, const opts *o, waste_ctx **ctx)
      * Kimi-Linear. Worth it for a container that was copied or downloaded
      * and has not been read since. */
     cfg.verify_records = o->verify;
+    cfg.exclusive_open = o->exclusive_open;
     const waste_status st = waste_open(path, &cfg, ctx);
     /* Two statuses that say nothing useful on their own when --cpus is
      * what produced them, and it usually is: nothing else here can be
@@ -283,6 +285,10 @@ static waste_status open_model(const char *path, const opts *o, waste_ctx **ctx)
     else if (o->cpus && st == WASTE_E_UNSUPPORTED)
         fprintf(stderr, "--cpus: this platform does not bind threads to "
                         "CPUs (Linux and Windows only)\n");
+    else if (o->exclusive_open && st == WASTE_E_BUSY)
+        fprintf(stderr, "--exclusive-open: another process owns this "
+                        "container; stop it or retry without "
+                        "--exclusive-open\n");
     return st;
 }
 
@@ -1220,6 +1226,7 @@ int main(int argc, char **argv)
                "options: --budget 8G  --ctx N  -n N  --temp F  --top-p F\n"
                "         --top-k N  --seed N  --threads N  --cpus LIST\n"
                "         --stop STR  --file F  --json  -q  --learn  --verify\n"
+               "         --exclusive-open\n"
          "  --stop  ends generation when the text appears\n"
          "  --json  machine-readable output for eval, tokenize, plan,\n"
          "          info and bench\n"
@@ -1231,6 +1238,7 @@ int main(int argc, char **argv)
          "  the cores differ: on a two-die Ryzen, six threads on one die\n"
          "  measured 16-25%% faster than six split across both. Linux and\n"
          "  Windows; the default is to leave placement to the OS\n"
+         "  --exclusive-open asks for single-process container ownership\n"
          "  --verify checks each expert record's checksum as it is read,\n"
          "  for a container you have not read since copying it. Costs ~5%%\n"
          "  on Kimi-Linear, ~1%% on K3; off otherwise\n",

docs/ENGINE.md

@@ -23,6 +23,30 @@ state save/load, model introspection and aggregate stats.
 Deliberately *not* in the API: logging to stdout, signal handlers, config
 files, argument parsing. Those belong to the host — the CLI included.
 
+### Optional container ownership
+
+On POSIX hosts, `waste_cfg.exclusive_open` asks `waste_open` to take a
+non-blocking advisory lock on the container directory before memory planning or
+model-sized allocation. A cooperating process that also requests exclusivity
+for the same container receives `WASTE_E_BUSY`; it does not wait. Paths are
+matched by device and inode, so aliases of one directory do not evade the
+check.
+
+Contexts in one process remain independent as documented: they share a
+reference-counted ownership entry, and the last `waste_close` releases it.
+Failures during planning, budget validation, or partial model loading release
+it as well. Lock descriptors are close-on-exec, and a forked child is treated
+as a different process rather than inheriting the parent's registry.
+
+Concurrent opens remain the default. Containers are read-only, and process
+ownership is host policy rather than a data-safety requirement; a workstation
+daemon can opt in through `waste_cfg.exclusive_open`, or `--exclusive-open` on
+the CLI and server. This is an advisory lock between cooperating WASTE
+processes, not RAM accounting or a security boundary. If the directory cannot
+be opened for locking or the filesystem does not support `flock`, the model
+continues without ownership. Windows keeps its existing lifecycle behavior and
+ignores the setting.
+
 ## 2. CLI as a first-class client
 
 `cli/` links the library and adds only host concerns: argv parsing, a

docs/SERVE.md

@@ -292,6 +292,7 @@ python3 -m serve MODEL [options]
   --vision           load the vision tower
   --verify           check every expert record's crc32 as it is read
   --usage PATH       learned hotlist (default <model>/usage.waste)
+  --exclusive-open    ask for POSIX single-process container ownership
   --max-tokens N     default cap when a request does not set one (4096)
   --no-thinking      answer without the think channel unless asked
   --allow-local-images

serve/__main__.py

@@ -23,7 +23,7 @@
 
 from . import api                                            # noqa: E402
 from .engine import (CACHE_LFRU, CACHE_LRU,                  # noqa: E402
-                     WASTE_E_ARG, WASTE_E_UNSUPPORTED,
+                     WASTE_E_ARG, WASTE_E_BUSY, WASTE_E_UNSUPPORTED,
                      Engine, EngineError, build_info, physical_ram,
                      plan_memory)
 from .server import serve                                    # noqa: E402
@@ -121,6 +121,8 @@ def main(argv=None) -> int:
                         "since. Costs ~5%% on Kimi-Linear, ~1%% on K3")
     g.add_argument("--usage", default=None, metavar="PATH",
                    help="learned hotlist (default <model>/usage.waste)")
+    g.add_argument("--exclusive-open", action="store_true",
+                   help="ask for single-process ownership of this container")
 
     s = ap.add_argument_group("serving")
     s.add_argument("--max-tokens", type=bounded_int(1, (1 << 32) - 1),
@@ -177,7 +179,8 @@ def main(argv=None) -> int:
             direct_io=not args.no_direct_io,
             vision=args.vision,
             verify_records=args.verify,
-            usage_path=args.usage)
+            usage_path=args.usage,
+            exclusive_open=args.exclusive_open)
     except EngineError as e:
         print(f"{e}", file=sys.stderr)
         # Two statuses that say nothing useful on their own when --cpus is
@@ -187,6 +190,9 @@ def main(argv=None) -> int:
         elif args.cpus and e.status == WASTE_E_UNSUPPORTED:
             print("--cpus: this platform does not bind threads to CPUs "
                   "(Linux and Windows only)", file=sys.stderr)
+        elif args.exclusive_open and e.status == WASTE_E_BUSY:
+            print("--exclusive-open: another process owns this container; "
+                  "stop it or retry without --exclusive-open", file=sys.stderr)
         return 1
 
     try:

serve/engine.py

@@ -44,6 +44,7 @@
 WASTE_E_ARG = -5
 WASTE_E_UNSUPPORTED = -6
 WASTE_E_CANCELLED = -7
+WASTE_E_BUSY = -8
 
 # waste.h's waste_cache_policy. There is no third: a "pinned" policy was
 # listed there and never implemented, so it selected LFRU like everything
@@ -87,7 +88,8 @@ class Cfg(C.Structure):
                 ("use_direct_io", C.c_int),
                 ("vision", C.c_int),
                 ("verify_records", C.c_int),
-                ("usage_path", C.c_char_p)]
+                ("usage_path", C.c_char_p),
+                ("exclusive_open", C.c_int)]
 
 
 class GenParams(C.Structure):
@@ -378,7 +380,8 @@ def __init__(self, model_path: str, *,
                  direct_io: bool = True,
                  vision: bool = False,
                  verify_records: bool = False,
-                 usage_path: Optional[str] = None):
+                 usage_path: Optional[str] = None,
+                 exclusive_open: bool = False):
         ram_budget_bytes = _bounded_int(
             "ram_budget_bytes", ram_budget_bytes, 0, (1 << 64) - 1)
         ctx_tokens = _bounded_int("ctx_tokens", ctx_tokens, 0, (1 << 32) - 1)
@@ -408,6 +411,7 @@ def __init__(self, model_path: str, *,
         # and a temporary would be freed before waste_open reads it.
         self._usage = usage_path.encode() if usage_path else None
         cfg.usage_path = self._usage
+        cfg.exclusive_open = 1 if exclusive_open else 0
 
         st = self.lib.waste_open(self.model_path.encode(), C.byref(cfg),
                                  C.byref(self._ctx))