diff --git a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml index a5596b19b..0bbcda73e 100644 --- a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml +++ b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml @@ -22,9 +22,9 @@ spec: requests: storage: 10Gi --- -# L3 tier for the fs secondary tier -- the vLLM analogue of SGLang's HiCache -# file backend. Ported from qwen36-27b-vllm.yaml: proven pattern, same -# hardware, same GPU-memory ceiling. +# L3 tier for the fs secondary tier -- the vLLM analogue of the HiCache file +# backend on the retired SGLang engine. Carried over from qwen36-27b-vllm.yaml +# (deleted in 5770d6902): proven pattern, same hardware, same GPU-memory ceiling. apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -180,8 +180,9 @@ spec: resourceName: squat.ai/dri --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/inferenceservice_v1alpha1.json -# concsweep.py, warm, agg tok/s vs the SGLang qwen-3.6 baseline (conc 1/8/16): -# 14.88/53.93/100.82 vs 14.96/36.24/34.49. Bench warm only -- cold reads ~12. +# concsweep.py, warm, agg tok/s vs the qwen-3.6 SGLang engine this replaced +# (retired 2026-08-16, conc 1/8/16): 14.88/53.93/100.82 vs 14.96/36.24/34.49. +# Bench warm only -- cold reads ~12. apiVersion: inference.llmkube.dev/v1alpha1 kind: InferenceService metadata: @@ -198,12 +199,11 @@ spec: # # The "before" row originally recorded here (31.06 28.64 39.30 52.15 52.22 # 61.99) was WITHDRAWN 2026-09-02: its M=2 value was a cold rep. This engine - # is bimodal until warmed -- cold reps land ~29-32 at M=2 where warm reps land - # ~55, a 1.9x gap with no traffic and no config change, and ONE warmup run is - # not enough. Isolated warm A/B of the LDS gate (8 reps, same pod, gate the - # only variable) gives M=2 43.74 -> 55.5 (+26.5%). M=3 was unchanged, but that - # is a null control, not a confirmation: at M=3 K*M=52224 still exceeds 39321 - # so down_proj stays on Triton and no movement was predicted. + # is bimodal until warmed (cold ~29-32 at M=2 vs warm ~55, a 1.9x gap with no + # traffic and no config change), so ONE warmup run is not enough. Isolated + # warm A/B of the LDS gate (8 reps, same pod, gate the only variable): M=2 + # 43.74 -> 55.5 (+26.5%). M=3 unchanged -- a null control, not a confirmation: + # at M=3 K*M=52224 still exceeds 39321, so down_proj stays on Triton. # # No warm number is claimed for the capture-size change itself. It is kept on # mechanism only: at M=6 every W4A16 projection falls to Triton @@ -216,7 +216,9 @@ spec: # as immutable as a commit tag -- but Renovate can re-resolve the tag and # open a digest-bump PR, which it can never do for an immutable # nightly- tag (no ordering scheme, so no update is ever detected). - # v0.27.1 is the newest release and predates gfx1201 support. + # A nightly, not a tagged release: gfx1201 support landed after v0.27.1 and + # this runs 0.28.1rc1.dev388. Whether the v0.28.0 tag (2026-08-26) alone now + # suffices has NOT been retested -- assume not until someone boots it. image: vllm/vllm-openai-rocm:nightly@sha256:0d07767c0b8471eaca0f1eca97899620dfb4ad33ff2f5226789afef1a877e9bc modelCache: claimName: qwen38-27b-vllm-model-cache @@ -249,7 +251,7 @@ spec: maxModelLen: 246944 kvCacheDtype: fp8_e4m3 # Confirmed working on this hybrid Mamba/GDN model (~27% hit rate under - # real traffic on the sibling qwen36 config). + # real traffic on the qwen36 config this replaced). enablePrefixCaching: true # Do NOT lower this to buy transcode headroom: 2048 was tried and cost # 4-16x TTFT (p50 5s -> 80s, avg 13.8s -> 58.5s) because a 20K-token @@ -261,14 +263,14 @@ spec: # sane value. The media-reserve math lives with that flag, not here. gpuMemoryUtilization: 0.875 env: - # Attention IS AITER while the 09-01 soak patch below is mounted: it lifts - # the supports_kv_connector rejection at rocm.py:703, and the boot log - # confirms "Overriding with ROCM_AITER_UNIFIED_ATTN". Drop that patch and a - # KV connector forces TRITON_ATTN again. FP8 linear kernels do not apply to + # Attention IS AITER while the patch below is mounted: it lifts the + # supports_kv_connector rejection at rocm.py:703, and the boot log confirms + # "Overriding with ROCM_AITER_UNIFIED_ATTN". Drop that patch and a KV + # connector forces TRITON_ATTN again. FP8 linear kernels do not apply to # W4A16. Also kept for the GDN linear-attn path. # An older forced-AITER experiment measured +84% decode / ~10x worse - # prefill; the soak block below measures parity. Trust the soak block -- - # the older number predates vllm#53821. + # prefill; the block below measures parity. Trust that block -- the older + # number predates vllm#53821. - name: VLLM_ROCM_USE_AITER value: "1" - name: HIP_VISIBLE_DEVICES @@ -301,7 +303,8 @@ spec: emptyDir: medium: Memory sizeLimit: 24Gi - # SOAK TEST -- see the mounts below and models/resources/zz_aiter_kvconn_impl.py. + # Pins ROCM_AITER_UNIFIED_ATTN -- see the mounts below and + # models/resources/zz_aiter_kvconn_impl.py. - name: aiter-kvconn-patch configMap: name: aiter-kvconn-patch @@ -316,29 +319,42 @@ spec: mountPath: /kvoffload - name: dshm mountPath: /dev/shm - # SOAK TEST 2026-09-01, time-boxed -- remove or promote after evaluation. - # EXIT CRITERION (added 09-03, was missing): the claimed win is short - # context, so compare litellm TTFT and ITL for prompts under 2K tokens - # over the 7d before 09-01 against the 7d after. Promote on a real - # improvement there, revert on parity -- the synthetic sweeps already - # measured parity at long context, so "no change" is a revert, not a pass. - # # Selects ROCM_AITER_UNIFIED_ATTN instead of TRITON_ATTN by lifting an # inherited `supports_kv_connector() -> False` that appears to be a missing # override rather than a real exclusion (full argument in the .py). # - # Bench says PARITY, not a win: decode conc-16 77.27 vs 77.63 tok/s, engine - # prefill 6388.5 vs 6390. It is deployed anyway to get real-traffic hours, - # because the published gains (vllm#43615: +56.5% at 512 tok, +72.4% on - # 1K-2K) live at short contexts that the synthetic sweeps did not cover, - # and production traffic does include them. + # ADOPTED 2026-09-04, promoted from the 09-01 soak. Kept for DETERMINISM, + # not speed: against TRITON_ATTN it measures parity (09-02 warm, decode M=2 + # 55.31 vs 55.72, M=3 64.07 vs 64.88 -- inside the within-arm spread). + # Without this patch a KV connector forces TRITON_ATTN; with it the backend + # survives a nightly reordering its preference list, which 0d07767 already + # does -- it now prefers TURBOQUANT and falls back to here. + # + # TURBOQUANT is not a candidate while the KV offload connector is on, and + # the reason is NOT the 09-01 gfx1201 CK FMHA segfault: it declares only + # KVCacheLayout.LBNHC (turboquant_attn.py), OffloadingConnector requires + # LBHNC (offloading_connector.py:188), and this backend advertises both + # LBHNC and LHBNC. Using it means dropping --kv-transfer-config, which + # measured a HALVING of single-stream decode (31 -> 15.5 tok/s). Note vLLM + # logs the actual rejection reasons at DEBUG only (rocm.py:680), so the + # INFO line naming AttentionType.DECODER is context, not the cause -- + # TurboQuant does support DECODER. Boot with VLLM_LOGGING_LEVEL=DEBUG to + # read the real reason rather than inferring it from layouts as above. + # The soak's short-context exit criterion was never measured and is now + # unmeasurable -- unrelated config changes land on both sides of 09-01. + # + # AITER v0.1.19 -> v0.1.21.post1 (image 0d07767), 4 warm reps each: decode + # M=2 55.31 -> 57.47 (Welch t 6.19), M=3 64.07 -> 66.71 (t 4.50). Uncached + # prefill unchanged (M<=1, t 0.87). Combined AITER+vLLM delta -- the Triton + # control arm was not re-run, so none of it is attributed to AITER alone. # # An earlier run measured 1.89 tok/s decode and looked catastrophic. That was # vllm#53821 (AITER's query_start_loc zeroed during graph capture), fixed in # this image. Do NOT reintroduce that number as evidence. # - # Verify it actually took effect -- the hook fails open to TRITON_ATTN: - # kubectl logs deploy/qwen38-27b-vllm -c vllm | grep aiter-kvconn-patch + # Both hooks fail open to TRITON_ATTN, so confirm the patch FIRED, not just + # that it installed -- only the first line below proves it took effect: + # kubectl logs deploy/qwen38-27b-vllm -c vllm | grep supports_kv_connector # kubectl logs deploy/qwen38-27b-vllm -c vllm | grep "Overriding with" # Revert = delete these two mounts, the volume above, and the generator. - name: aiter-kvconn-patch @@ -396,7 +412,7 @@ spec: # M=5 as 6 -- the M=5 point was measuring the all-Triton M=6 path. With # max_num_seqs 5 there is no padding left to pay for. - '{"cudagraph_capture_sizes": [1, 2, 3, 4, 5]}' - # Keep prior-turn reasoning in context; matches production qwen36 config. + # Keep prior-turn reasoning in context; carried over from the qwen36 config. - --default-chat-template-kwargs - '{"preserve_thinking": true}' # MTP speculative decoding is deliberately OMITTED: measured on this @@ -443,9 +459,9 @@ spec: - "0.05" - --kv-cache-memory - "10200547328" - # Hierarchical KV cache: CPU tier (matches SGLang's --hicache-ratio sizing - # philosophy) + fs tier (SGLang's file backend analogue). Ported from - # qwen36-27b-vllm.yaml, WITHOUT that config's --language-model-only — + # Hierarchical KV cache: CPU tier (sizing philosophy carried over from the + # retired SGLang engine's --hicache-ratio) + fs tier (its file-backend + # analogue). From qwen36-27b-vllm.yaml, WITHOUT its --language-model-only — # Hermes' auxiliary.vision block actively uses this model's vision tower, # so it must stay loaded even though that costs some context headroom. # CPU tier 22Gi: the 16Gi tier cascaded 3.5 GB/hour to the fs tier, i.e. @@ -470,20 +486,18 @@ spec: # nested key is silently accepted and ignored, so verify after any edit by # grepping the boot log for that literal. # - # Tried 8 on 2026-09-03 and REVERTED the same evening. It did what it was - # designed to do -- chunk queries 13.13 -> 9.79 per request, hit rate flat - # at 40.5% vs 40.9% -- but each lookup got 44% slower (4.33s -> 6.24s) and - # the number that matters, TOTAL lookup wall time per request, went UP: - # 5.47s -> 6.46s, or +45% once normalised for the shorter prompts in the - # after-window (0.099 -> 0.143 s per 1K prompt tokens). + # Tried 8 on 2026-09-03, REVERTED the same evening: chunk queries fell + # 13.13 -> 9.79 per request and hit rate stayed flat (40.5% vs 40.9%), but + # each lookup got 44% slower (4.33s -> 6.24s) and total lookup wall time per + # request went UP, 5.47s -> 6.46s (+45% normalised for prompt length, + # 0.099 -> 0.143 s per 1K prompt tokens). # # Mechanism: the tier has ONE serial lookup thread and is page-cache-bound, - # not throughput-bound, so coalescing more blocks per lookup does not - # reduce serialisation, it just makes each serial unit bigger. Raising this - # cannot help for that reason -- do not retry 8 or 16 without first making - # the lookup manager concurrent. Judge any future attempt on total lookup - # seconds per request, never on the per-lookup mean, which improves by - # construction as the count falls. + # not throughput-bound -- coalescing more blocks per lookup makes each + # serial unit bigger without reducing serialisation. Do not retry 8 or 16 + # without first making the lookup manager concurrent, and judge any attempt + # on total lookup seconds per request, never the per-lookup mean (which + # improves by construction as the count falls). - --kv-transfer-config - >- {"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"spec_name":"TieringOffloadingSpec","cpu_bytes_to_use":23622320128,"blocks_per_chunk":4,"secondary_tiers":[{"type":"fs","root_dir":"/kvoffload","locality":"LOCAL"}]}}