From be0b9260c1917cda10ab552aea7b58d8ae17c54b Mon Sep 17 00:00:00 2001 From: Asher Feldman <59994+asher@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:47:28 -0700 Subject: [PATCH] fix(serve): explicit defaults no longer fork a duplicate resident model --- CHANGELOG.md | 3 +++ gmlx/config.py | 39 +++++++++++++++++++++------- gmlx/loader.py | 4 ++- tests/test_config.py | 60 +++++++++++++++++++++++++++++++++++++++----- 4 files changed, 90 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 528ac49..f6c9a15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed +- Server: two model ids over the same GGUF now share one resident copy when + they differ only in spelling a default explicitly (`moe_prestage: ranked`, + `prefill_feeder: true`, or any streaming lever on a model with no `stream`). - `--over-generation` runs now render thinking in the verbose stream like normal runs (the probe path bypassed the styled emitter). - CLI polish: `--help`/`--help-all` now win even after a value-taking flag, diff --git a/gmlx/config.py b/gmlx/config.py index 4994996..cd03b11 100644 --- a/gmlx/config.py +++ b/gmlx/config.py @@ -42,6 +42,7 @@ import yaml from . import profiles as _family_profiles +from .envflags import env_bool # Canonical key sets / env mappings # Sampling keys a profile may carry, as a plain dict so profiles compose by @@ -525,7 +526,33 @@ def load_signature(self) -> tuple: both load-affecting - the template is baked into the tokenizer and the adapter is wrapped over the model leaves at load, so profiles differing in either need their own resident entry. Sampling/system/ttl do not change the loaded model - and are excluded.""" + and are excluded. + + The stream-riding keys enter as their effective values, not their + config spellings, so an explicitly written default never forks a + resident entry from an unset key: without a ``stream`` placement the + MoE levers and feeder overrides are inert (announced as ignored at + load) and collapse to None; ``moe_prestage`` collapses to None + whenever it resolves to ranked behavior (``ranked``, or ``keepers`` + without ``moe_miss_shed``); the feeder tri-states resolve through the + same explicit-then-env-then-default policy as + loader._resolve_feeder_defaults. The env reads happen in the serving + process, which is also where the load happens, so signature and load + always see the same values.""" + stream = self.stream or None + prestage = self.moe_prestage if self.moe_prestage == "keepers" else None + if self.moe_miss_shed is None: + prestage = None + if stream: + pf = (self.prefill_feeder if self.prefill_feeder is not None + else env_bool("GMLX_FEEDER_PREFILL", True)) + df = (self.decode_feeder if self.decode_feeder is not None + else env_bool("GMLX_FEEDER_DECODE", stream == "experts")) + levers = (str(self.moe_experts), str(self.moe_expert_mass), + str(self.moe_miss_shed), str(self.moe_layer_shed), + str(prestage), str(pf), str(df)) + else: + levers = (str(None),) * 7 return ( self.path, self.mmproj, @@ -534,14 +561,8 @@ def load_signature(self) -> tuple: str(self.speculative_width_cap), self.chat_template, self.adapter, - str(self.stream), - str(self.moe_experts), - str(self.moe_expert_mass), - str(self.moe_miss_shed), - str(self.moe_layer_shed), - str(self.moe_prestage), - str(self.prefill_feeder), - str(self.decode_feeder), + str(stream), + *levers, tuple(sorted((k, str(v)) for k, v in self.load.items())), tuple(sorted((k, str(v)) for k, v in _flatten_cache(self.cache).items())), ) diff --git a/gmlx/loader.py b/gmlx/loader.py index 8f7ae3b..df2a450 100644 --- a/gmlx/loader.py +++ b/gmlx/loader.py @@ -1283,7 +1283,9 @@ def _resolve_feeder_defaults( defaults: prefill feeder on everywhere it can exist, decode feeder on only when the every-token layers are on the GPU (``--stream-experts``) - under ``--stream-cpu`` there is no GPU work for the arena gathers to - join, so it is not even attempted.""" + join, so it is not even attempted. config.ResolvedModel.load_signature + mirrors this resolution to canonicalize the residency cache key; keep + the two in step.""" gpu_resident = "gpu" in str(mx.default_device()).lower() if feeder_prefill is None: feeder_prefill = env_bool("GMLX_FEEDER_PREFILL", True) diff --git a/tests/test_config.py b/tests/test_config.py index 5ee876c..5814ad0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -253,12 +253,19 @@ def test_moe_lossy_levers_parsed_and_resolved(): speculative=False, mmproj=None, draft_gguf=None, pin=False, ttl_s=None, stream="experts") base_sig = cfgmod.ResolvedModel(id="x", **common).load_signature() - for key, val in (("moe_experts", 2), ("moe_expert_mass", 0.5), - ("moe_miss_shed", 0.5), ("moe_layer_shed", 0.5), - ("moe_prestage", "keepers")): - sig = cfgmod.ResolvedModel( - id="x", **{key: val}, **common).load_signature() - assert sig != base_sig, key + # (keepers needs a shed policy to take effect, so its arm carries one and + # is compared against the shed-only signature, not the bare base.) + for extra in ({"moe_experts": 2}, {"moe_expert_mass": 0.5}, + {"moe_miss_shed": 0.5}, {"moe_layer_shed": 0.5}, + {"moe_miss_shed": 0.5, "moe_prestage": "keepers"}): + sig = cfgmod.ResolvedModel(id="x", **extra, **common).load_signature() + assert sig != base_sig, extra + shed_only = cfgmod.ResolvedModel( + id="x", moe_miss_shed=0.5, **common).load_signature() + with_keepers = cfgmod.ResolvedModel( + id="x", moe_miss_shed=0.5, moe_prestage="keepers", + **common).load_signature() + assert with_keepers != shed_only for key, bad in (("moe_experts", 0), ("moe_experts", "many"), ("moe_miss_shed", 1.5), ("moe_miss_shed", 0), @@ -271,6 +278,47 @@ def test_moe_lossy_levers_parsed_and_resolved(): build_config(doc) +def test_load_signature_canonicalizes_effective_defaults(monkeypatch): + """The signature hashes effective values, not config spellings: writing + a default explicitly must not fork a resident entry from an unset key.""" + monkeypatch.delenv("GMLX_FEEDER_PREFILL", raising=False) + monkeypatch.delenv("GMLX_FEEDER_DECODE", raising=False) + common = dict(path="/p", sampling={}, load={}, cache={}, system=None, + speculative=False, mmproj=None, draft_gguf=None, pin=False, + ttl_s=None) + + def sig(**kw): + return cfgmod.ResolvedModel(id="x", **common, **kw).load_signature() + + base = sig(stream="experts") + # Explicit spellings of the streaming defaults collapse to the unset form. + assert sig(stream="experts", moe_prestage="ranked") == base + assert sig(stream="experts", prefill_feeder=True) == base + assert sig(stream="experts", decode_feeder=True) == base + # keepers without a shed policy is announced-ignored: also the unset form. + assert sig(stream="experts", moe_prestage="keepers") == base + # Real differences still fork. + assert sig(stream="experts", prefill_feeder=False) != base + assert sig(stream="experts", decode_feeder=False) != base + # Under stream: cpu the decode feeder defaults off, so True forks and + # False collapses (mirrors loader._resolve_feeder_defaults). + cpu = sig(stream="cpu") + assert sig(stream="cpu", decode_feeder=True) != cpu + assert sig(stream="cpu", decode_feeder=False) == cpu + # Without a placement every stream-riding key is inert (announced as + # ignored at load) and the signature treats them as unset. + plain = sig() + for kw in ({"moe_experts": 4}, {"moe_expert_mass": 0.9}, + {"moe_miss_shed": 0.7}, {"moe_layer_shed": 0.1}, + {"moe_miss_shed": 0.7, "moe_prestage": "keepers"}, + {"prefill_feeder": False}, {"decode_feeder": False}): + assert sig(**kw) == plain, kw + # The env A/B levers move the effective default; the signature follows. + monkeypatch.setenv("GMLX_FEEDER_PREFILL", "0") + assert sig(stream="experts", prefill_feeder=False) == sig(stream="experts") + assert sig(stream="experts", prefill_feeder=True) != sig(stream="experts") + + def test_stream_legacy_cpu_moe_alias_warns_and_maps(): """`cpu_moe:` still works (old semantics: hybrid -> experts, true/full -> cpu) but warns about the rename; an explicit `stream:` wins silently."""