diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 418ad5ab9..41d12d251 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -589,6 +589,98 @@ kimik2.5-int4-mi355x-vllm: search-space: - { tp: 8, conc-start: 4, conc-end: 64 } +kimik2.5-mxfp4-mi355x-vllm-eagle3: + image: vllm/vllm-openai-rocm:v0.21.0 + model: amd/Kimi-K2.5-MXFP4 + model-prefix: kimik2.5 + runner: mi355x + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + +kimik2.5-int4-mi355x-vllm-eagle3: + image: vllm/vllm-openai-rocm:v0.21.0 + model: moonshotai/Kimi-K2.5 + model-prefix: kimik2.5 + runner: mi355x + precision: int4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + +kimik2.5-int4-mi355x-vllm-fixed-ar-mtp: + image: vllm/vllm-openai-rocm:v0.21.0 + model: moonshotai/Kimi-K2.5 + model-prefix: kimik2.5 + runner: mi355x + precision: int4 + framework: vllm + multinode: false + scenarios: + fixed-ar-mtp: + - isl: 1024 + osl: 1024 + draft-model: nvidia/Kimi-K2.5-Thinking-Eagle3 + num-speculative-tokens: 3 + rejection-sample-method: synthetic + synthetic-acceptance-rates: [0.778774, 0.57543, 0.412793] + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + - isl: 8192 + osl: 1024 + draft-model: nvidia/Kimi-K2.5-Thinking-Eagle3 + num-speculative-tokens: 3 + rejection-sample-method: synthetic + synthetic-acceptance-rates: [0.778774, 0.57543, 0.412793] + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + +kimik2.5-fp4-mi355x-vllm-fixed-ar-mtp: + image: vllm/vllm-openai-rocm:v0.21.0 + model: amd/Kimi-K2.5-MXFP4 + model-prefix: kimik2.5 + runner: mi355x + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-ar-mtp: + - isl: 1024 + osl: 1024 + draft-model: lightseekorg/kimi-k2.5-eagle3 + num-speculative-tokens: 3 + rejection-sample-method: synthetic + synthetic-acceptance-rates: [0.778774, 0.57543, 0.412793] + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + - isl: 8192 + osl: 1024 + draft-model: lightseekorg/kimi-k2.5-eagle3 + num-speculative-tokens: 3 + rejection-sample-method: synthetic + synthetic-acceptance-rates: [0.778774, 0.57543, 0.412793] + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + kimik2.5-int4-mi325x-vllm: image: vllm/vllm-openai-rocm:v0.21.0 model: moonshotai/Kimi-K2.5 @@ -724,6 +816,25 @@ minimaxm2.5-fp8-mi355x-vllm: - { tp: 4, ep: 4, conc-start: 4, conc-end: 512 } - { tp: 8, ep: 8, conc-start: 2, conc-end: 2 } +minimaxm2.5-fp8-mi355x-vllm-eagle3: + image: vllm/vllm-openai-rocm:v0.21.0 + model: MiniMaxAI/MiniMax-M2.5 + model-prefix: minimaxm2.5 + runner: mi355x + precision: fp8 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + # Diverged from minimaxm2.5-fp8-mi355x-vllm (agentic-coding sibling). Reasons below; # the original minimaxm2.5-fp8-mi355x-vllm entry is left identical to origin/main so # its fixed-seq-len sweep is unaffected. diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index cca6031c3..82153570e 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -53,7 +53,6 @@ on: run-eval: type: boolean required: true - default: false eval-only: description: "Run only evals (skip throughput benchmark)" type: boolean @@ -68,10 +67,30 @@ on: required: false type: string scenario-type: - description: "Scenario type (fixed-seq-len or agentic-coding)" + description: "Scenario type (fixed-seq-len, agentic-coding, or fixed-ar-mtp)" required: false type: string default: 'fixed-seq-len' + draft-model: + description: "Draft model for fixed-AR MTP scenarios" + required: false + type: string + default: '' + num-speculative-tokens: + description: "Number of speculative tokens for fixed-AR MTP scenarios" + required: false + type: string + default: '' + rejection-sample-method: + description: "Speculative rejection sampling method" + required: false + type: string + default: '' + synthetic-acceptance-rates: + description: "JSON array of synthetic acceptance rates for fixed-AR MTP scenarios" + required: false + type: string + default: '' offloading: description: "KV offload backend for agentic scenarios (none/cpu/ssd)" required: false @@ -111,6 +130,10 @@ env: SCENARIO_TYPE: ${{ inputs.scenario-type }} SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || '' }} IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }} + DRAFT_MODEL: ${{ inputs.draft-model }} + NUM_SPECULATIVE_TOKENS: ${{ inputs.num-speculative-tokens }} + REJECTION_SAMPLE_METHOD: ${{ inputs.rejection-sample-method }} + SYNTHETIC_ACCEPTANCE_RATES: ${{ inputs.synthetic-acceptance-rates }} OFFLOADING: ${{ inputs.offloading }} TOTAL_CPU_DRAM_GB: ${{ inputs.total-cpu-dram-gb }} DURATION: ${{ inputs.duration }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index fea89fcae..290c503ba 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -51,6 +51,7 @@ jobs: multi-node-eval-config: ${{ steps.get-jobs.outputs.multi-node-eval-config }} agentic-config: ${{ steps.get-jobs.outputs.agentic-config }} multi-node-agentic-config: ${{ steps.get-jobs.outputs.multi-node-agentic-config }} + fixed-ar-mtp-config: ${{ steps.get-jobs.outputs.fixed-ar-mtp-config }} steps: - name: Checkout code (ref) if: ${{ inputs.ref && inputs.ref != '' }} @@ -71,12 +72,14 @@ jobs: ${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}) AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x]))") MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))") - SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))") + FIXED_AR_MTP=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'fixed-ar-mtp']))") + SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') not in ('agentic-coding', 'fixed-ar-mtp') and not x.get('eval-only', False)]))") MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))") EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))") MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))") echo "agentic-config=$AGENTIC" >> $GITHUB_OUTPUT echo "multi-node-agentic-config=$MULTI_AGENTIC" >> $GITHUB_OUTPUT + echo "fixed-ar-mtp-config=$FIXED_AR_MTP" >> $GITHUB_OUTPUT echo "single-node-config=$SINGLE" >> $GITHUB_OUTPUT echo "multi-node-config=$MULTI" >> $GITHUB_OUTPUT echo "eval-config=$EVALS" >> $GITHUB_OUTPUT @@ -190,7 +193,7 @@ jobs: osl: '0' max-model-len: '0' spec-decoding: 'none' - disagg: 'false' + disagg: ${{ 'false' }} run-eval: false scenario-type: agentic-coding ref: ${{ inputs.ref }} @@ -235,6 +238,41 @@ jobs: scenario-type: agentic-coding ref: ${{ inputs.ref }} + test-sweep-fixed-ar-mtp: + needs: get-jobs + if: ${{ needs.get-jobs.outputs.fixed-ar-mtp-config != '[]' }} + uses: ./.github/workflows/benchmark-tmpl.yml + name: Fixed-AR-MTP throughput / + strategy: + fail-fast: false + matrix: + config: ${{ fromJson(needs.get-jobs.outputs.fixed-ar-mtp-config) }} + secrets: inherit + with: + exp-name: ${{ matrix.config.exp-name }} + isl: ${{ matrix.config.isl }} + osl: ${{ matrix.config.osl }} + max-model-len: ${{ matrix.config.max-model-len }} + runner: ${{ matrix.config.runner }} + image: ${{ matrix.config.image }} + model: ${{ matrix.config.model }} + model-prefix: ${{ matrix.config.model-prefix }} + framework: ${{ matrix.config.framework }} + precision: ${{ matrix.config.precision }} + tp: ${{ matrix.config.tp }} + ep: ${{ matrix.config.ep }} + dp-attn: ${{ matrix.config.dp-attn }} + conc: ${{ matrix.config.conc }} + spec-decoding: ${{ matrix.config.spec-decoding }} + disagg: ${{ matrix.config.disagg }} + run-eval: false + scenario-type: fixed-ar-mtp + draft-model: ${{ matrix.config.draft-model }} + num-speculative-tokens: ${{ matrix.config.num-speculative-tokens }} + rejection-sample-method: ${{ matrix.config.rejection-sample-method }} + synthetic-acceptance-rates: ${{ toJson(matrix.config.synthetic-acceptance-rates) }} + ref: ${{ inputs.ref }} + test-sweep-single-node: needs: get-jobs if: ${{ needs.get-jobs.outputs.single-node-config != '[]' }} @@ -297,8 +335,8 @@ jobs: ref: ${{ inputs.ref }} collect-results: - needs: [test-sweep-multi-node, test-sweep-single-node, test-sweep-agentic, test-sweep-multi-node-agentic] - if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped') }} + needs: [test-sweep-multi-node, test-sweep-single-node, test-sweep-fixed-ar-mtp, test-sweep-agentic, test-sweep-multi-node-agentic] + if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-fixed-ar-mtp.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped') }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: diff --git a/.github/workflows/mtp-fixed-ar-amd.yml b/.github/workflows/mtp-fixed-ar-amd.yml new file mode 100644 index 000000000..3f6d0a391 --- /dev/null +++ b/.github/workflows/mtp-fixed-ar-amd.yml @@ -0,0 +1,619 @@ +name: MTP benchmark with fixed AR +run-name: MTP fixed AR - ${{ inputs.test-name || inputs.model }} + +on: + workflow_dispatch: + inputs: + test-name: + description: "Display name for this run" + required: false + type: string + default: "lcb-v6-kimik2.5-MXFP4-mtp-fixed-ar" + runner: + description: "MI355X self-hosted runner to use" + required: true + type: string + default: "mi355x-amds_0" + image: + description: "ROCm vLLM Docker image" + required: true + type: string + default: "vllm/vllm-openai-rocm:v0.21.0" + model: + description: "Target model served by vLLM" + required: true + type: string + default: "amd/Kimi-K2.5-MXFP4" + draft-model: + description: "Draft model used by Eagle3 speculative decoding" + required: true + type: string + default: "lightseekorg/kimi-k2.5-eagle3" + dataset: + description: "Hugging Face dataset used for acceptance-rate sampling" + required: true + type: string + default: "livecodebench/code_generation_lite" + dataset-config: + description: "Optional Hugging Face dataset config/subset name" + required: false + type: string + default: "" + dataset-split: + description: "Dataset split to load. Empty means test if present, otherwise first split." + required: false + type: string + default: "test6" + dataset-prompt-fields: + description: "Comma-separated candidate prompt fields, supports dotted paths" + required: false + type: string + default: "question_content" + prompt-turns: + description: "For list-based prompts such as MT-Bench turns: first or all" + required: true + type: choice + options: + - first + - all + default: "first" + num-prompts: + description: "Number of prompts to sample from the dataset" + required: true + type: string + default: "200" + max-concurrency: + description: "vLLM bench serve max concurrency. Supports a single value or comma-separated values, e.g. 32,64" + required: true + type: string + default: "32,64" + port: + description: "vLLM server port" + required: true + type: string + default: "8000" + num-speculative-tokens: + description: "Number of speculative tokens" + required: true + type: string + default: "3" + tensor-parallel-size: + description: "Tensor parallel size for vLLM serve" + required: true + type: string + default: "8" + draft-tensor-parallel-size: + description: "Draft model tensor parallel size for Eagle3 speculative decoding" + required: true + type: string + default: "1" + vllm-env-json: + description: "Optional JSON object of extra environment variables for vLLM serve" + required: false + type: string + default: '{"VLLM_ROCM_USE_AITER":"1","VLLM_ROCM_QUICK_REDUCE_QUANTIZATION":"INT4","VLLM_ROCM_USE_AITER_RMSNORM":"0"}' + vllm-extra-args: + description: "Optional extra arguments appended to vLLM serve" + required: false + type: string + default: "--enable-expert-parallel --long-prefill-token-threshold 8192 --max-num-batched-tokens 16384" + bench-extra-args: + description: "Optional extra arguments appended to vLLM bench serve" + required: false + type: string + default: "" + +permissions: + contents: read + +jobs: + generate-synthetic-acceptance-rates: + runs-on: ${{ inputs.runner }} + timeout-minutes: 500 + outputs: + synthetic-acceptance-rates: ${{ steps.compute-rates.outputs.synthetic_acceptance_rates }} + speculative-config: ${{ steps.compute-rates.outputs.speculative_config }} + + env: + CONTAINER_NAME: mtp-fixed-ar-${{ github.run_id }}-${{ github.run_attempt }} + HOST_WORK_DIR: /tmp/mtp-fixed-ar-${{ github.run_id }}-${{ github.run_attempt }} + CONTAINER_WORK_DIR: /work/tmp/mtp-fixed-ar-${{ github.run_id }}-${{ github.run_attempt }} + DATASET_JSONL: dataset_prompts.jsonl + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Prepare workspace + run: | + set -euo pipefail + mkdir -p "$HOST_WORK_DIR" + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + + - name: Launch ROCm vLLM container + run: | + set -euo pipefail + docker run -d -it \ + --ipc=host \ + --network=host \ + --privileged \ + --cap-add=CAP_SYS_ADMIN \ + --device=/dev/kfd \ + --device=/dev/dri \ + --device=/dev/mem \ + --group-add video \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --shm-size 32G \ + -e HF_TOKEN \ + -e HUGGING_FACE_HUB_TOKEN \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + -v /:/work \ + --entrypoint "/bin/bash" \ + --name "$CONTAINER_NAME" \ + "${{ inputs.image }}" + + - name: Download dataset and convert to vLLM JSONL + env: + DATASET: ${{ inputs.dataset }} + DATASET_CONFIG: ${{ inputs.dataset-config }} + DATASET_SPLIT: ${{ inputs.dataset-split }} + DATASET_PROMPT_FIELDS: ${{ inputs.dataset-prompt-fields }} + PROMPT_TURNS: ${{ inputs.prompt-turns }} + NUM_PROMPTS: ${{ inputs.num-prompts }} + run: | + set -euo pipefail + docker exec \ + -e HF_TOKEN \ + -e HUGGING_FACE_HUB_TOKEN \ + -e DATASET \ + -e DATASET_CONFIG \ + -e DATASET_SPLIT \ + -e DATASET_PROMPT_FIELDS \ + -e PROMPT_TURNS \ + -e NUM_PROMPTS \ + -e CONTAINER_WORK_DIR \ + -e DATASET_JSONL \ + "$CONTAINER_NAME" \ + bash -lc ' + set -euo pipefail + mkdir -p "$CONTAINER_WORK_DIR" + python3 -m pip install -q --no-cache-dir datasets huggingface_hub || \ + python3 -m pip install -q --no-cache-dir --break-system-packages datasets huggingface_hub + hf download "$DATASET" --repo-type=dataset >/dev/null + python3 - <<'"'"'PY'"'"' + import json + import os + from pathlib import Path + from datasets import get_dataset_split_names, load_dataset + from huggingface_hub import snapshot_download + + dataset_name = os.environ["DATASET"] + dataset_config = os.environ.get("DATASET_CONFIG") or None + requested_split = os.environ.get("DATASET_SPLIT") or None + prompt_fields = [ + field.strip() + for field in os.environ.get("DATASET_PROMPT_FIELDS", "").split(",") + if field.strip() + ] + prompt_turns = os.environ.get("PROMPT_TURNS", "first") + num_prompts = int(os.environ["NUM_PROMPTS"]) + out_path = os.path.join(os.environ["CONTAINER_WORK_DIR"], os.environ["DATASET_JSONL"]) + + split_kwargs = {"path": dataset_name} + load_kwargs = {"path": dataset_name} + if dataset_config: + split_kwargs["name"] = dataset_config + load_kwargs["name"] = dataset_config + + def get_path(row, path): + value = row + for part in path.split("."): + if isinstance(value, dict) and part in value: + value = value[part] + else: + return None + return value + + def message_text(message): + if isinstance(message, str): + return message + if isinstance(message, dict): + content = message.get("content") or message.get("value") or message.get("text") + if isinstance(content, str): + return content + return None + + def normalize_prompt(value): + if isinstance(value, str) and value.strip(): + return value + if isinstance(value, list): + texts = [] + for item in value: + text = message_text(item) + if text and text.strip(): + texts.append(text) + if texts: + return "\n\n".join(texts) if prompt_turns == "all" else texts[0] + if isinstance(value, dict): + for key in ("prompt", "content", "value", "text"): + prompt = normalize_prompt(value.get(key)) + if prompt: + return prompt + return None + + def load_rows_from_downloaded_files(): + snapshot = Path(snapshot_download(dataset_name, repo_type="dataset")) + split_name = requested_split or "test" + candidates = [snapshot / f"{split_name}.jsonl", snapshot / f"{split_name}.json"] + if not requested_split: + candidates.extend(sorted(snapshot.glob("*.jsonl"))) + candidates.extend(sorted(snapshot.glob("*.json"))) + + data_file = next((path for path in candidates if path.is_file()), None) + if data_file is None: + raise FileNotFoundError( + f"No JSON/JSONL data file found in downloaded dataset snapshot: {snapshot}" + ) + + rows = [] + with data_file.open(encoding="utf-8") as fh: + if data_file.suffix == ".jsonl": + for line in fh: + if line.strip(): + rows.append(json.loads(line)) + if len(rows) >= num_prompts: + break + else: + data = json.load(fh) + if isinstance(data, dict): + for key in ("data", "rows", "examples"): + if isinstance(data.get(key), list): + data = data[key] + break + if not isinstance(data, list): + raise ValueError(f"Unsupported JSON dataset shape in {data_file}") + rows = data[:num_prompts] + return rows, data_file.name + + try: + splits = get_dataset_split_names(**split_kwargs) + split = requested_split or ("test" if "test" in splits else splits[0]) + dataset = load_dataset(**load_kwargs, split=split) + rows = dataset.select(range(min(num_prompts, len(dataset)))) + source = f"split={split}" + except RuntimeError as exc: + if "Dataset scripts are no longer supported" not in str(exc): + raise + rows, source = load_rows_from_downloaded_files() + + rows = list(rows) + if rows and len(rows) < num_prompts: + rows = [rows[i % len(rows)] for i in range(num_prompts)] + + # vLLM's custom dataset mode consumes JSONL rows with a prompt field. + # This replaces a fixed path like /work/.../livecodebench_v6_prompts.jsonl. + with open(out_path, "w", encoding="utf-8") as out: + for row in rows: + prompt = None + for field in prompt_fields: + prompt = normalize_prompt(get_path(row, field)) + if prompt: + break + if prompt is None: + prompt = json.dumps(row, ensure_ascii=False) + out.write(json.dumps({"prompt": prompt}, ensure_ascii=False) + "\n") + + print(f"Wrote prompts from {dataset_name} {source} to {out_path}") + PY + ' + + - name: Launch vLLM server with standard speculative decoding + env: + MODEL: ${{ inputs.model }} + DRAFT_MODEL: ${{ inputs.draft-model }} + PORT: ${{ inputs.port }} + NUM_SPECULATIVE_TOKENS: ${{ inputs.num-speculative-tokens }} + TENSOR_PARALLEL_SIZE: ${{ inputs.tensor-parallel-size }} + DRAFT_TENSOR_PARALLEL_SIZE: ${{ inputs.draft-tensor-parallel-size }} + VLLM_ENV_JSON: ${{ inputs.vllm-env-json }} + VLLM_EXTRA_ARGS: ${{ inputs.vllm-extra-args }} + run: | + set -euo pipefail + docker exec -d \ + -e MODEL \ + -e DRAFT_MODEL \ + -e PORT \ + -e NUM_SPECULATIVE_TOKENS \ + -e TENSOR_PARALLEL_SIZE \ + -e DRAFT_TENSOR_PARALLEL_SIZE \ + -e VLLM_ENV_JSON \ + -e VLLM_EXTRA_ARGS \ + -e CONTAINER_WORK_DIR \ + "$CONTAINER_NAME" \ + bash -lc ' + set -euo pipefail + DEFAULT_DRAFT_MODEL="lightseekorg/kimi-k2.5-eagle3" + DEFAULT_VLLM_ENV_JSON='"'"'{"VLLM_ROCM_USE_AITER":"1","VLLM_ROCM_QUICK_REDUCE_QUANTIZATION":"INT4","VLLM_ROCM_USE_AITER_RMSNORM":"0"}'"'"' + DEFAULT_VLLM_EXTRA_ARGS="--enable-expert-parallel --long-prefill-token-threshold 8192 --max-num-batched-tokens 16384" + if [[ "$MODEL" == "moonshotai/Kimi-K2.5" || "$MODEL" == "amd/Kimi-K2.5-MXFP4" ]]; then + if [[ -z "${VLLM_EXTRA_ARGS:-}" ]]; then + VLLM_EXTRA_ARGS="$DEFAULT_VLLM_EXTRA_ARGS" + fi + fi + if [[ "$MODEL" == "MiniMaxAI/MiniMax-M2.5" ]]; then + if [[ "$DRAFT_MODEL" == "$DEFAULT_DRAFT_MODEL" ]]; then + DRAFT_MODEL="thoughtworks/MiniMax-M2.5-Eagle3" + fi + if [[ "${VLLM_ENV_JSON:-}" == "$DEFAULT_VLLM_ENV_JSON" ]]; then + VLLM_ENV_JSON='"'"'{"VLLM_ROCM_USE_AITER":"1","VLLM_ROCM_QUICK_REDUCE_QUANTIZATION":"INT4"}'"'"' + fi + if [[ -z "${VLLM_EXTRA_ARGS:-}" || "${VLLM_EXTRA_ARGS:-}" == "$DEFAULT_VLLM_EXTRA_ARGS" ]]; then + VLLM_EXTRA_ARGS="--enable-expert-parallel --kv-cache-dtype fp8 --block-size=32 --attention-backend ROCM_AITER_FA" + fi + fi + python3 -m pip install -q --no-cache-dir amd-quark || \ + python3 -m pip install -q --no-cache-dir --break-system-packages amd-quark + SPECULATIVE_CONFIG=$(python3 - <<'"'"'PY'"'"' + import json + import os + + print(json.dumps({ + "model": os.environ["DRAFT_MODEL"], + "method": "eagle3", + "num_speculative_tokens": int(os.environ["NUM_SPECULATIVE_TOKENS"]), + "rejection_sample_method": "standard", + "draft_tensor_parallel_size": int(os.environ["DRAFT_TENSOR_PARALLEL_SIZE"]), + })) + PY + ) + while IFS= read -r export_line; do + export "$export_line" + done < <(python3 - <<'"'"'PY'"'"' + import json + import os + + env = json.loads(os.environ.get("VLLM_ENV_JSON") or "{}") + if not isinstance(env, dict): + raise SystemExit("vllm-env-json must be a JSON object") + for key, value in env.items(): + print(f"{key}={value}") + PY + ) + vllm serve "$MODEL" \ + --host 0.0.0.0 \ + --port "$PORT" \ + -tp "$TENSOR_PARALLEL_SIZE" \ + --mm-encoder-tp-mode data \ + --no-enable-prefix-caching \ + --speculative-config "$SPECULATIVE_CONFIG" \ + --trust-remote-code \ + $VLLM_EXTRA_ARGS \ + > "$CONTAINER_WORK_DIR/vllm_server.log" 2>&1 + ' + + - name: Wait for vLLM server + env: + PORT: ${{ inputs.port }} + run: | + set -euo pipefail + for i in {1..180}; do + if curl --silent --fail "http://127.0.0.1:${PORT}/health" >/dev/null; then + exit 0 + fi + docker exec "$CONTAINER_NAME" bash -lc "tail -n 80 '$CONTAINER_WORK_DIR/vllm_server.log' 2>/dev/null || true" + sleep 10 + done + echo "vLLM server did not become healthy" >&2 + exit 1 + + - name: Run LiveCodeBench sampling benchmark + env: + MODEL: ${{ inputs.model }} + PORT: ${{ inputs.port }} + NUM_PROMPTS: ${{ inputs.num-prompts }} + MAX_CONCURRENCY: ${{ inputs.max-concurrency }} + BENCH_EXTRA_ARGS: ${{ inputs.bench-extra-args }} + run: | + set -euo pipefail + docker exec \ + -e MODEL \ + -e PORT \ + -e NUM_PROMPTS \ + -e MAX_CONCURRENCY \ + -e BENCH_EXTRA_ARGS \ + -e CONTAINER_WORK_DIR \ + -e DATASET_JSONL \ + "$CONTAINER_NAME" \ + bash -lc ' + set -euo pipefail + combined_log="$CONTAINER_WORK_DIR/vllm_bench_serve.log" + : > "$combined_log" + + IFS="," read -ra concurrency_values <<< "$MAX_CONCURRENCY" + ran_any=false + for raw_concurrency in "${concurrency_values[@]}"; do + concurrency="${raw_concurrency//[[:space:]]/}" + if [[ -z "$concurrency" ]]; then + continue + fi + if [[ ! "$concurrency" =~ ^[0-9]+$ ]]; then + echo "Invalid max-concurrency value: $raw_concurrency" >&2 + exit 1 + fi + + log_file="$CONTAINER_WORK_DIR/vllm_bench_serve_conc${concurrency}.log" + ran_any=true + { + echo "===== max-concurrency=${concurrency} =====" + echo "dataset=$CONTAINER_WORK_DIR/$DATASET_JSONL" + echo "num-prompts=$NUM_PROMPTS" + echo "" + } > "$log_file" + + if ! vllm bench serve \ + --backend openai \ + --base-url "http://127.0.0.1:${PORT}" \ + --endpoint /v1/completions \ + --model "$MODEL" \ + --dataset-name custom \ + --dataset-path "$CONTAINER_WORK_DIR/$DATASET_JSONL" \ + --num-prompts "$NUM_PROMPTS" \ + --max-concurrency "$concurrency" \ + --request-rate inf \ + --temperature 0.0 \ + --ignore-eos \ + --trust-remote-code \ + $BENCH_EXTRA_ARGS \ + >> "$log_file" 2>&1; then + cat "$log_file" >> "$combined_log" + exit 1 + fi + + cat "$log_file" >> "$combined_log" + echo "" >> "$combined_log" + done + + if [[ "$ran_any" != "true" ]]; then + echo "No max-concurrency values were provided." >&2 + exit 1 + fi + ' + + - name: Capture vLLM metrics + env: + PORT: ${{ inputs.port }} + run: | + set -euo pipefail + curl --silent --fail "http://127.0.0.1:${PORT}/metrics" -o "$HOST_WORK_DIR/livecodebench_metrics.txt" + + - name: Compute synthetic acceptance rates + id: compute-rates + env: + MODEL: ${{ inputs.model }} + DRAFT_MODEL: ${{ inputs.draft-model }} + NUM_SPECULATIVE_TOKENS: ${{ inputs.num-speculative-tokens }} + run: | + set -euo pipefail + python3 - <<'PY' | tee "$HOST_WORK_DIR/synthetic_acceptance_rates.log" + import json + import os + import re + from collections import defaultdict + from pathlib import Path + + path = Path(os.environ["HOST_WORK_DIR"]) / "livecodebench_metrics.txt" + + drafts_by_metric = defaultdict(float) + accepted_by_metric = defaultdict(lambda: defaultdict(float)) + + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + + parts = line.split() + if len(parts) < 2: + continue + + series, value = parts[0], float(parts[1]) + metric = series.split("{", 1)[0] + + if metric in ("vllm:spec_decode_num_drafts_total", "vllm:spec_decode_num_drafts"): + drafts_by_metric[metric] += value + + if metric in ( + "vllm:spec_decode_num_accepted_tokens_per_pos_total", + "vllm:spec_decode_num_accepted_tokens_per_pos", + ): + match = re.search(r'position="(\d+)"', series) + if match: + accepted_by_metric[metric][int(match.group(1))] += value + + draft_metric = ( + "vllm:spec_decode_num_drafts_total" + if drafts_by_metric["vllm:spec_decode_num_drafts_total"] > 0 + else "vllm:spec_decode_num_drafts" + ) + accepted_metric = ( + "vllm:spec_decode_num_accepted_tokens_per_pos_total" + if accepted_by_metric["vllm:spec_decode_num_accepted_tokens_per_pos_total"] + else "vllm:spec_decode_num_accepted_tokens_per_pos" + ) + + num_drafts = drafts_by_metric[draft_metric] + accepted = accepted_by_metric[accepted_metric] + num_speculative_tokens = int(os.environ["NUM_SPECULATIVE_TOKENS"]) + + if num_drafts <= 0: + raise SystemExit("No spec decode drafts found. Did you run standard speculative decoding first?") + + rates = [accepted[i] / num_drafts for i in range(num_speculative_tokens)] + rounded_rates = [round(rate, 6) for rate in rates] + draft_model = os.environ["DRAFT_MODEL"] + if ( + os.environ["MODEL"] == "MiniMaxAI/MiniMax-M2.5" + and draft_model == "lightseekorg/kimi-k2.5-eagle3" + ): + draft_model = "thoughtworks/MiniMax-M2.5-Eagle3" + + speculative_config = { + "method": "eagle3", + "model": draft_model, + "num_speculative_tokens": num_speculative_tokens, + "rejection_sample_method": "synthetic", + "synthetic_acceptance_rates": rounded_rates, + } + + out_dir = Path(os.environ["HOST_WORK_DIR"]) + (out_dir / "synthetic_acceptance_rates.json").write_text( + json.dumps({ + "num_drafts": int(num_drafts), + "accepted_per_position": {str(i): int(accepted[i]) for i in range(num_speculative_tokens)}, + "synthetic_acceptance_rates": rounded_rates, + "speculative_config": speculative_config, + }, indent=2) + "\n", + encoding="utf-8", + ) + + print("num_drafts =", int(num_drafts)) + print("accepted_per_position =", {i: int(accepted[i]) for i in range(num_speculative_tokens)}) + print("synthetic_acceptance_rates =", json.dumps(rounded_rates)) + print("speculative_config =", json.dumps(speculative_config)) + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as fh: + fh.write(f"synthetic_acceptance_rates={json.dumps(rounded_rates)}\n") + fh.write(f"speculative_config={json.dumps(speculative_config)}\n") + PY + + - name: Add workflow summary + run: | + set -euo pipefail + { + echo "## Synthetic Acceptance Rates" + echo "" + echo '```text' + cat "$HOST_WORK_DIR/synthetic_acceptance_rates.log" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload acceptance-rate artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mtp-fixed-ar-artifacts + path: | + ${{ env.HOST_WORK_DIR }}/${{ env.DATASET_JSONL }} + ${{ env.HOST_WORK_DIR }}/livecodebench_metrics.txt + ${{ env.HOST_WORK_DIR }}/synthetic_acceptance_rates.json + ${{ env.HOST_WORK_DIR }}/synthetic_acceptance_rates.log + ${{ env.HOST_WORK_DIR }}/vllm_server.log + ${{ env.HOST_WORK_DIR }}/vllm_bench_serve*.log + if-no-files-found: ignore + + - name: Cleanup container + if: always() + run: | + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true diff --git a/benchmarks/single_node/kimik2.5_fp4_mi355x_vllm_eagle3.sh b/benchmarks/single_node/kimik2.5_fp4_mi355x_vllm_eagle3.sh new file mode 100755 index 000000000..e5bf04e24 --- /dev/null +++ b/benchmarks/single_node/kimik2.5_fp4_mi355x_vllm_eagle3.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + CONC \ + ISL \ + OSL \ + MAX_MODEL_LEN \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi + +# Install amd-quark for MXFP4 quantization support. +pip install amd-quark + +# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ +if [ -n "$ROCR_VISIBLE_DEVICES" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +SERVER_LOG=/workspace/server.log +PORT=${PORT:-8888} + +if [ "${EVAL_ONLY}" = "true" ]; then + setup_eval_context + MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi + +SPECULATIVE_CONFIG=$(python3 - <<'PY' +import json +import os + +config = { + "model": os.environ.get("DRAFT_MODEL", "lightseekorg/kimi-k2.5-eagle3"), + "method": "eagle3", + "num_speculative_tokens": int(os.environ.get("NUM_SPECULATIVE_TOKENS", "3")), + "rejection_sample_method": "standard", + "draft_tensor_parallel_size": int(os.environ.get("DRAFT_TENSOR_PARALLEL_SIZE", "1")), +} +print(json.dumps(config)) +PY +) + +# Start GPU monitoring (power, temperature, clocks every second) +start_gpu_monitor + +set -x +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_USE_AITER_RMSNORM=0 + +vllm serve "$MODEL" --port "$PORT" \ +--tensor-parallel-size="$TP" \ +--enable-expert-parallel \ +--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.90}" \ +--max-model-len "$MAX_MODEL_LEN" \ +--max-num-seqs "$CONC" \ +--long-prefill-token-threshold 8192 \ +--max-num-batched-tokens 16384 \ +--mm-encoder-tp-mode data \ +--no-enable-prefix-caching \ +--speculative-config "$SPECULATIVE_CONFIG" \ +--trust-remote-code > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +# Wait for server to be ready +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --trust-remote-code \ + --server-pid "$SERVER_PID" + +# After throughput, run evaluation only if RUN_EVAL is true +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +# Stop GPU monitoring +stop_gpu_monitor +set +x diff --git a/benchmarks/single_node/kimik2.5_fp4_mi355x_vllm_mtp_fixed_AR.sh b/benchmarks/single_node/kimik2.5_fp4_mi355x_vllm_mtp_fixed_AR.sh new file mode 100755 index 000000000..7472e2cd5 --- /dev/null +++ b/benchmarks/single_node/kimik2.5_fp4_mi355x_vllm_mtp_fixed_AR.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + DRAFT_MODEL \ + NUM_SPECULATIVE_TOKENS \ + REJECTION_SAMPLE_METHOD \ + SYNTHETIC_ACCEPTANCE_RATES \ + TP \ + CONC \ + ISL \ + OSL \ + MAX_MODEL_LEN \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi + +# Install amd-quark for MXFP4 quantization support. +pip install amd-quark + +# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ +if [ -n "$ROCR_VISIBLE_DEVICES" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +SERVER_LOG=/workspace/server.log +PORT=${PORT:-8888} + +if [ "${EVAL_ONLY}" = "true" ]; then + setup_eval_context + MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi + +SPECULATIVE_CONFIG=$(python3 - <<'PY' +import json +import os + +print(json.dumps({ + "method": "eagle3", + "model": os.environ["DRAFT_MODEL"], + "num_speculative_tokens": int(os.environ["NUM_SPECULATIVE_TOKENS"]), + # Matrix/config fields use kebab-case; vLLM expects snake_case JSON. + "rejection_sample_method": os.environ["REJECTION_SAMPLE_METHOD"], + "synthetic_acceptance_rates": json.loads(os.environ["SYNTHETIC_ACCEPTANCE_RATES"]), + "draft_tensor_parallel_size": int(os.environ.get("DRAFT_TENSOR_PARALLEL_SIZE", "1")), +})) +PY +) + +# Start GPU monitoring (power, temperature, clocks every second) +start_gpu_monitor + +set -x +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_USE_AITER_RMSNORM=0 + +vllm serve "$MODEL" --port "$PORT" \ +--tensor-parallel-size="$TP" \ +--enable-expert-parallel \ +--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.90}" \ +--max-model-len "$MAX_MODEL_LEN" \ +--max-num-seqs "$CONC" \ +--long-prefill-token-threshold 8192 \ +--max-num-batched-tokens 16384 \ +--mm-encoder-tp-mode data \ +--no-enable-prefix-caching \ +--speculative-config "$SPECULATIVE_CONFIG" \ +--trust-remote-code > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +# Wait for server to be ready +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --trust-remote-code \ + --server-pid "$SERVER_PID" + +# After throughput, run evaluation only if RUN_EVAL is true +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +# Stop GPU monitoring +stop_gpu_monitor +set +x diff --git a/benchmarks/single_node/kimik2.5_int4_mi355x_vllm_eagle3.sh b/benchmarks/single_node/kimik2.5_int4_mi355x_vllm_eagle3.sh new file mode 100755 index 000000000..68345dca2 --- /dev/null +++ b/benchmarks/single_node/kimik2.5_int4_mi355x_vllm_eagle3.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + CONC \ + ISL \ + OSL \ + MAX_MODEL_LEN \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi + +# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ +if [ -n "$ROCR_VISIBLE_DEVICES" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +SERVER_LOG=/workspace/server.log +PORT=${PORT:-8888} + +if [ "${EVAL_ONLY}" = "true" ]; then + setup_eval_context + MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi + +SPECULATIVE_CONFIG=$(python3 - <<'PY' +import json +import os + +config = { + "model": os.environ.get("DRAFT_MODEL", "lightseekorg/kimi-k2.5-eagle3"), + "method": "eagle3", + "num_speculative_tokens": int(os.environ.get("NUM_SPECULATIVE_TOKENS", "3")), + "rejection_sample_method": "standard", + "draft_tensor_parallel_size": int(os.environ.get("DRAFT_TENSOR_PARALLEL_SIZE", "1")), +} +print(json.dumps(config)) +PY +) + +# Start GPU monitoring (power, temperature, clocks every second) +start_gpu_monitor + +set -x +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_USE_AITER_RMSNORM=0 + +vllm serve "$MODEL" --port "$PORT" \ +--tensor-parallel-size="$TP" \ +--enable-expert-parallel \ +--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.90}" \ +--max-model-len "$MAX_MODEL_LEN" \ +--max-num-seqs "$CONC" \ +--long-prefill-token-threshold 8192 \ +--max-num-batched-tokens 16384 \ +--mm-encoder-tp-mode data \ +--no-enable-prefix-caching \ +--speculative-config "$SPECULATIVE_CONFIG" \ +--trust-remote-code > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +# Wait for server to be ready +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --trust-remote-code \ + --server-pid "$SERVER_PID" + +# After throughput, run evaluation only if RUN_EVAL is true +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +# Stop GPU monitoring +stop_gpu_monitor +set +x diff --git a/benchmarks/single_node/kimik2.5_int4_mi355x_vllm_mtp_fixed_AR.sh b/benchmarks/single_node/kimik2.5_int4_mi355x_vllm_mtp_fixed_AR.sh new file mode 100755 index 000000000..8a782427d --- /dev/null +++ b/benchmarks/single_node/kimik2.5_int4_mi355x_vllm_mtp_fixed_AR.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + DRAFT_MODEL \ + NUM_SPECULATIVE_TOKENS \ + REJECTION_SAMPLE_METHOD \ + SYNTHETIC_ACCEPTANCE_RATES \ + TP \ + CONC \ + ISL \ + OSL \ + MAX_MODEL_LEN \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi + +# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ +if [ -n "$ROCR_VISIBLE_DEVICES" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +SERVER_LOG=/workspace/server.log +PORT=${PORT:-8888} + +if [ "${EVAL_ONLY}" = "true" ]; then + setup_eval_context + MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi + +SPECULATIVE_CONFIG=$(python3 - <<'PY' +import json +import os + +print(json.dumps({ + "method": "eagle3", + "model": os.environ["DRAFT_MODEL"], + "num_speculative_tokens": int(os.environ["NUM_SPECULATIVE_TOKENS"]), + # Matrix/config fields use kebab-case; vLLM expects snake_case JSON. + "rejection_sample_method": os.environ["REJECTION_SAMPLE_METHOD"], + "synthetic_acceptance_rates": json.loads(os.environ["SYNTHETIC_ACCEPTANCE_RATES"]), +})) +PY +) + +# Start GPU monitoring (power, temperature, clocks every second) +start_gpu_monitor + +set -x +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_USE_AITER_RMSNORM=0 + +vllm serve "$MODEL" --port "$PORT" \ +--tensor-parallel-size="$TP" \ +--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.90}" \ +--max-model-len "$MAX_MODEL_LEN" \ +--trust-remote-code \ +--no-enable-prefix-caching \ +--max-num-seqs "$CONC" \ +--mm-encoder-tp-mode data \ +--speculative-config "$SPECULATIVE_CONFIG" > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +# Wait for server to be ready +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --trust-remote-code \ + --server-pid "$SERVER_PID" + +# After throughput, run evaluation only if RUN_EVAL is true +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +# Stop GPU monitoring +stop_gpu_monitor +set +x diff --git a/benchmarks/single_node/minimaxm2.5_fp8_mi355x_vllm_eagle3.sh b/benchmarks/single_node/minimaxm2.5_fp8_mi355x_vllm_eagle3.sh new file mode 100755 index 000000000..9c5c6fb51 --- /dev/null +++ b/benchmarks/single_node/minimaxm2.5_fp8_mi355x_vllm_eagle3.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + CONC \ + ISL \ + OSL \ + MAX_MODEL_LEN \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi + +# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ +if [ -n "$ROCR_VISIBLE_DEVICES" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +SERVER_LOG=/workspace/server.log +PORT=${PORT:-8888} +ASYNC_SCHEDULING_ARGS="" +VLLM_BLOCK_SIZE=32 + +if [ "${EVAL_ONLY}" = "true" ]; then + setup_eval_context + MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi + +SPECULATIVE_CONFIG=$(python3 - <<'PY' +import json +import os + +config = { + "model": os.environ.get("DRAFT_MODEL", "thoughtworks/MiniMax-M2.5-Eagle3"), + "method": "eagle3", + "num_speculative_tokens": int(os.environ.get("NUM_SPECULATIVE_TOKENS", "3")), + "rejection_sample_method": "standard", + "draft_tensor_parallel_size": int(os.environ.get("DRAFT_TENSOR_PARALLEL_SIZE", "1")), +} +print(json.dumps(config)) +PY +) + +# Start GPU monitoring (power, temperature, clocks every second) +start_gpu_monitor + +set -x +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 + +if [[ "$ISL" == "8192" && "$OSL" == "1024" && "$TP" == "8" ]]; then + export VLLM_ROCM_USE_AITER_MOE=0 + ASYNC_SCHEDULING_ARGS="--no-async-scheduling" + echo "8k1k TP8: disabling AITER MoE and async scheduling to avoid ROCm resource exhaustion." + if (( CONC >= 64 )); then + VLLM_BLOCK_SIZE=16 + echo "8k1k TP8 c${CONC}: using block size 16." + fi +fi + +vllm serve "$MODEL" --port "$PORT" \ +--tensor-parallel-size="$TP" \ +--enable-expert-parallel \ +--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.90}" \ +--max-model-len "$MAX_MODEL_LEN" \ +--max-num-seqs "$CONC" \ +--kv-cache-dtype fp8 \ +--block-size="$VLLM_BLOCK_SIZE" \ +--no-enable-prefix-caching \ +--attention-backend "ROCM_AITER_FA" \ +$ASYNC_SCHEDULING_ARGS \ +--speculative-config "$SPECULATIVE_CONFIG" \ +--trust-remote-code > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +# Wait for server to be ready +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --trust-remote-code \ + --server-pid "$SERVER_PID" + +# After throughput, run evaluation only if RUN_EVAL is true +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +# Stop GPU monitoring +stop_gpu_monitor +set +x diff --git a/benchmarks/single_node/minimaxm2.5_fp8_mi355x_vllm_eagle3_fixed_AR.sh b/benchmarks/single_node/minimaxm2.5_fp8_mi355x_vllm_eagle3_fixed_AR.sh new file mode 100755 index 000000000..ed585f239 --- /dev/null +++ b/benchmarks/single_node/minimaxm2.5_fp8_mi355x_vllm_eagle3_fixed_AR.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + DRAFT_MODEL \ + NUM_SPECULATIVE_TOKENS \ + REJECTION_SAMPLE_METHOD \ + SYNTHETIC_ACCEPTANCE_RATES \ + TP \ + CONC \ + ISL \ + OSL \ + MAX_MODEL_LEN \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi + +# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ +if [ -n "$ROCR_VISIBLE_DEVICES" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +SERVER_LOG=/workspace/server.log +PORT=${PORT:-8888} +ASYNC_SCHEDULING_ARGS="" +VLLM_BLOCK_SIZE=32 + +if [ "${EVAL_ONLY}" = "true" ]; then + setup_eval_context + MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi + +SPECULATIVE_CONFIG=$(python3 - <<'PY' +import json +import os + +print(json.dumps({ + "model": os.environ["DRAFT_MODEL"], + "method": "eagle3", + "num_speculative_tokens": int(os.environ["NUM_SPECULATIVE_TOKENS"]), + # Matrix/config fields use kebab-case; vLLM expects snake_case JSON. + "rejection_sample_method": os.environ["REJECTION_SAMPLE_METHOD"], + "synthetic_acceptance_rates": json.loads(os.environ["SYNTHETIC_ACCEPTANCE_RATES"]), + "draft_tensor_parallel_size": int(os.environ.get("DRAFT_TENSOR_PARALLEL_SIZE", "1")), +})) +PY +) + +# Start GPU monitoring (power, temperature, clocks every second) +start_gpu_monitor + +set -x +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 + +if [[ "$ISL" == "8192" && "$OSL" == "1024" && "$TP" == "8" ]]; then + export VLLM_ROCM_USE_AITER_MOE=0 + ASYNC_SCHEDULING_ARGS="--no-async-scheduling" + echo "8k1k TP8: disabling AITER MoE and async scheduling to avoid ROCm resource exhaustion." + if (( CONC >= 64 )); then + VLLM_BLOCK_SIZE=16 + echo "8k1k TP8 c${CONC}: using block size 16." + fi +fi + +vllm serve "$MODEL" --port "$PORT" \ +--tensor-parallel-size="$TP" \ +--enable-expert-parallel \ +--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.90}" \ +--max-model-len "$MAX_MODEL_LEN" \ +--max-num-seqs "$CONC" \ +--kv-cache-dtype fp8 \ +--block-size="$VLLM_BLOCK_SIZE" \ +--no-enable-prefix-caching \ +--attention-backend "ROCM_AITER_FA" \ +$ASYNC_SCHEDULING_ARGS \ +--speculative-config "$SPECULATIVE_CONFIG" \ +--trust-remote-code > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +# Wait for server to be ready +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --trust-remote-code \ + --server-pid "$SERVER_PID" + +# After throughput, run evaluation only if RUN_EVAL is true +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +# Stop GPU monitoring +stop_gpu_monitor +set +x diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 789e363cb..9dbbc50c6 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -180,7 +180,17 @@ else export PORT_OFFSET=${RUNNER_NAME: -1} export PORT=$(( 8888 + ${PORT_OFFSET} )) FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "atom" ]] && printf '_atom' || printf '') - SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') + case "$SPEC_DECODING" in + mtp) + SPEC_SUFFIX="_mtp" + ;; + eagle3) + SPEC_SUFFIX="_eagle3" + ;; + *) + SPEC_SUFFIX="" + ;; + esac PARTITION="compute" SQUASH_FILE="/var/lib/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" @@ -224,8 +234,14 @@ else SCRIPT_BASE="${EXP_NAME%%_*}_${PRECISION}_mi355x" SCRIPT_FW="benchmarks/single_node/${SCENARIO_SUBDIR:-}${SCRIPT_BASE}_${FRAMEWORK}${SPEC_SUFFIX}.sh" + SCRIPT_FIXED_AR="benchmarks/single_node/${SCENARIO_SUBDIR:-}${SCRIPT_BASE}_${FRAMEWORK}${SPEC_SUFFIX}_fixed_AR.sh" + SCRIPT_FIXED_AR_MTP="benchmarks/single_node/${SCENARIO_SUBDIR:-}${SCRIPT_BASE}_${FRAMEWORK}_mtp_fixed_AR.sh" SCRIPT_FALLBACK="benchmarks/single_node/${SCENARIO_SUBDIR:-}${SCRIPT_BASE}${FRAMEWORK_SUFFIX}${SPEC_SUFFIX}.sh" - if [[ -f "$SCRIPT_FW" ]]; then + if [[ "$SCENARIO_TYPE" == "fixed-ar-mtp" && -f "$SCRIPT_FIXED_AR" ]]; then + BENCHMARK_SCRIPT="$SCRIPT_FIXED_AR" + elif [[ "$SCENARIO_TYPE" == "fixed-ar-mtp" && -f "$SCRIPT_FIXED_AR_MTP" ]]; then + BENCHMARK_SCRIPT="$SCRIPT_FIXED_AR_MTP" + elif [[ -f "$SCRIPT_FW" ]]; then BENCHMARK_SCRIPT="$SCRIPT_FW" else BENCHMARK_SCRIPT="$SCRIPT_FALLBACK" diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 9f38292f4..a43d2dc1a 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -26,6 +26,24 @@ seq_len_itos = {v: k for k, v in seq_len_stoi.items()} +def parse_concurrency_filter(values): + if values is None: + return None + + conc_values = [] + for value in values: + for part in str(value).split(","): + part = part.strip() + if not part: + continue + try: + conc_values.append(int(part)) + except ValueError as exc: + raise argparse.ArgumentTypeError( + f"invalid concurrency value: {part!r}") from exc + return conc_values + + def seq_len_to_str(isl: int, osl: int) -> str: """Convert sequence lengths to short string representation. @@ -124,7 +142,7 @@ def _max_eval_conc(ie): # Mark the selected entries (skip agentic entries which don't support evals) for i, entry in enumerate(matrix_values): - if entry.get(Fields.SCENARIO_TYPE.value) == 'agentic-coding': + if entry.get(Fields.SCENARIO_TYPE.value) in ('agentic-coding', 'fixed-ar-mtp'): continue entry[Fields.RUN_EVAL.value] = i in eval_indices if i in mn_eval_conc: @@ -187,6 +205,7 @@ def generate_full_sweep(args, all_config_data, runner_data): scenarios = val[Fields.SCENARIOS.value] scenario_filter = set(args.scenario_type) if getattr(args, 'scenario_type', None) else None seq_len_configs = scenarios.get(Fields.FIXED_SEQ_LEN.value, []) if (scenario_filter is None or 'fixed-seq-len' in scenario_filter) else [] + fixed_ar_mtp_configs = scenarios.get(Fields.FIXED_AR_MTP.value, []) if (scenario_filter is None or 'fixed-ar-mtp' in scenario_filter) else [] image = val[Fields.IMAGE.value] model = val[Fields.MODEL.value] precision = val[Fields.PRECISION.value] @@ -378,6 +397,97 @@ def generate_full_sweep(args, all_config_data, runner_data): if conc > conc_end: conc = conc_end + # ---- Fixed-AR MTP throughput scenarios ---- + if not is_multinode: + for fixed_ar_config in fixed_ar_mtp_configs: + isl = fixed_ar_config[Fields.ISL.value] + osl = fixed_ar_config[Fields.OSL.value] + + if seq_lens_filter and (isl, osl) not in seq_lens_filter: + continue + + bmk_space = fixed_ar_config[Fields.SEARCH_SPACE.value] + seq_len_str = seq_len_to_str(isl, osl) + runners_for_entry = runner_nodes_to_use if runner_nodes_to_use else [runner] + + for bmk in bmk_space: + tp = bmk[Fields.TP.value] + ep = bmk.get(Fields.EP.value) + dp_attn = bmk.get(Fields.DP_ATTN.value) + spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "mtp") + + if Fields.CONC_LIST.value in bmk: + conc_values = bmk[Fields.CONC_LIST.value] + else: + conc_start = bmk[Fields.CONC_START.value] + conc_end = bmk[Fields.CONC_END.value] + conc_values = [] + conc = conc_start + while conc <= conc_end: + conc_values.append(conc) + if conc == conc_end: + break + conc *= args.step_size + if conc > conc_end: + conc = conc_end + + if args.max_tp is not None: + if args.max_tp <= 0: + continue + if tp > args.max_tp: + continue + + if args.max_ep is not None: + if args.max_ep <= 0: + continue + if ep is not None and ep > args.max_ep: + ep = args.max_ep + + if args.min_conc is not None: + if args.min_conc <= 0: + continue + conc_values = [c for c in conc_values if c >= args.min_conc] + if not conc_values: + continue + + if args.max_conc is not None: + if args.max_conc <= 0: + continue + filtered_conc = [c for c in conc_values if c <= args.max_conc] + if not filtered_conc: + conc_values = [args.max_conc] + else: + conc_values = filtered_conc + + for conc in conc_values: + for runner_value in runners_for_entry: + entry = { + Fields.IMAGE.value: image, + Fields.MODEL.value: model, + Fields.MODEL_PREFIX.value: model_code, + Fields.PRECISION.value: precision, + Fields.FRAMEWORK.value: framework, + Fields.RUNNER.value: runner_value, + Fields.ISL.value: isl, + Fields.OSL.value: osl, + Fields.TP.value: tp, + Fields.CONC.value: conc, + Fields.MAX_MODEL_LEN.value: isl + osl + 256, + Fields.EP.value: ep if ep is not None else 1, + Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False, + Fields.SPEC_DECODING.value: spec_decoding, + Fields.EXP_NAME.value: f"{model_code}_{seq_len_str}", + Fields.DISAGG.value: disagg, + Fields.RUN_EVAL.value: False, + Fields.SCENARIO_TYPE.value: "fixed-ar-mtp", + Fields.DRAFT_MODEL.value: fixed_ar_config[Fields.DRAFT_MODEL.value], + Fields.NUM_SPECULATIVE_TOKENS.value: fixed_ar_config[Fields.NUM_SPECULATIVE_TOKENS.value], + Fields.REJECTION_SAMPLE_METHOD.value: fixed_ar_config[Fields.REJECTION_SAMPLE_METHOD.value], + Fields.SYNTHETIC_ACCEPTANCE_RATES.value: fixed_ar_config[Fields.SYNTHETIC_ACCEPTANCE_RATES.value], + } + validate_matrix_entry(entry, is_multinode=False) + matrix_values.append(entry) + # ---- Agentic-coding scenarios ---- agentic_configs = scenarios.get(Fields.AGENTIC_CODING.value, []) if (scenario_filter is None or 'agentic-coding' in scenario_filter) else [] @@ -682,6 +792,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): scenario_filter = set(args.scenario_type) if getattr(args, 'scenario_type', None) else None fixed_configs = val[Fields.SCENARIOS.value].get(Fields.FIXED_SEQ_LEN.value, []) if (scenario_filter is None or 'fixed-seq-len' in scenario_filter) else [] + fixed_ar_mtp_configs = val[Fields.SCENARIOS.value].get(Fields.FIXED_AR_MTP.value, []) if (scenario_filter is None or 'fixed-ar-mtp' in scenario_filter) else [] for seq_len_config in fixed_configs: isl = seq_len_config[Fields.ISL.value] osl = seq_len_config[Fields.OSL.value] @@ -794,6 +905,71 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): } matrix_values.append(validate_matrix_entry(entry, is_multinode=False)) + # ---- Fixed-AR MTP throughput scenarios ---- + if not is_multinode: + for fixed_ar_config in fixed_ar_mtp_configs: + isl = fixed_ar_config[Fields.ISL.value] + osl = fixed_ar_config[Fields.OSL.value] + + if seq_lens_filter and (isl, osl) not in seq_lens_filter: + continue + + seq_len_str = seq_len_to_str(isl, osl) + + for bmk in fixed_ar_config[Fields.SEARCH_SPACE.value]: + tp = bmk[Fields.TP.value] + ep = bmk.get(Fields.EP.value) + dp_attn = bmk.get(Fields.DP_ATTN.value) + spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "mtp") + + if Fields.CONC_LIST.value in bmk: + conc_values = bmk[Fields.CONC_LIST.value] + else: + conc_start = bmk[Fields.CONC_START.value] + conc_end = bmk[Fields.CONC_END.value] + conc_values = [] + conc = conc_start + while conc <= conc_end: + conc_values.append(conc) + if conc == conc_end: + break + conc *= 2 + if conc > conc_end: + conc = conc_end + + if getattr(args, 'conc', None): + conc_values = [c for c in conc_values if c in args.conc] + if not conc_values: + continue + + for conc in conc_values: + for runner_value in runners_for_entry: + entry = { + Fields.IMAGE.value: image, + Fields.MODEL.value: model, + Fields.MODEL_PREFIX.value: model_code, + Fields.PRECISION.value: precision, + Fields.FRAMEWORK.value: framework, + Fields.RUNNER.value: runner_value, + Fields.ISL.value: isl, + Fields.OSL.value: osl, + Fields.TP.value: tp, + Fields.CONC.value: conc, + Fields.MAX_MODEL_LEN.value: isl + osl + 256, + Fields.EP.value: ep if ep is not None else 1, + Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False, + Fields.SPEC_DECODING.value: spec_decoding, + Fields.EXP_NAME.value: f"{model_code}_{seq_len_str}", + Fields.DISAGG.value: disagg, + Fields.RUN_EVAL.value: False, + Fields.SCENARIO_TYPE.value: "fixed-ar-mtp", + Fields.DRAFT_MODEL.value: fixed_ar_config[Fields.DRAFT_MODEL.value], + Fields.NUM_SPECULATIVE_TOKENS.value: fixed_ar_config[Fields.NUM_SPECULATIVE_TOKENS.value], + Fields.REJECTION_SAMPLE_METHOD.value: fixed_ar_config[Fields.REJECTION_SAMPLE_METHOD.value], + Fields.SYNTHETIC_ACCEPTANCE_RATES.value: fixed_ar_config[Fields.SYNTHETIC_ACCEPTANCE_RATES.value], + } + matrix_values.append(validate_matrix_entry(entry, is_multinode=False)) + # ---- Agentic-coding scenarios ---- agentic_configs = val[Fields.SCENARIOS.value].get(Fields.AGENTIC_CODING.value, []) if (scenario_filter is None or 'agentic-coding' in scenario_filter) else [] for agentic_config in agentic_configs: @@ -947,7 +1123,7 @@ def main(): parent_parser.add_argument( '--scenario-type', nargs='+', - choices=['fixed-seq-len', 'agentic-coding'], + choices=['fixed-seq-len', 'agentic-coding', 'fixed-ar-mtp'], required=False, help='Scenario type(s) to include. If not specified, all scenario types are generated.' ) @@ -1116,9 +1292,8 @@ def main(): test_config_keys_parser.add_argument( '--conc', nargs='+', - type=int, required=False, - help='Only include these concurrency values. Values must exist in the config conc-range/list.' + help='Only include these concurrency values. Supports space-separated or comma-separated values, e.g. 32 64 or 32,64. Values must exist in the config conc-range/list.' ) test_config_keys_parser.add_argument( '--seq-lens', @@ -1134,6 +1309,8 @@ def main(): ) args = parser.parse_args() + if args.command == 'test-config': + args.conc = parse_concurrency_filter(args.conc) apply_node_type_defaults(args) # Load and validate configuration files (validation happens by default in load functions) diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index dd245aec7..53f30e2fb 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -26,6 +26,7 @@ class Fields(Enum): # Scenario type keys FIXED_SEQ_LEN = 'fixed-seq-len' AGENTIC_CODING = 'agentic-coding' + FIXED_AR_MTP = 'fixed-ar-mtp' # Seq-len-config fields ISL = 'isl' @@ -53,6 +54,12 @@ class Fields(Enum): OFFLOADING = 'offloading' DURATION = 'duration' + # Fixed acceptance-rate MTP fields + DRAFT_MODEL = 'draft-model' + NUM_SPECULATIVE_TOKENS = 'num-speculative-tokens' + REJECTION_SAMPLE_METHOD = 'rejection-sample-method' + SYNTHETIC_ACCEPTANCE_RATES = 'synthetic-acceptance-rates' + # Matrix entry fields CONC = 'conc' MAX_MODEL_LEN = 'max-model-len' @@ -86,7 +93,7 @@ class SingleNodeMatrixEntry(BaseModel): model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) precision: str framework: str - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "eagle3", "draft_model", "none"] = Field( alias=Fields.SPEC_DECODING.value ) runner: str @@ -103,6 +110,28 @@ class SingleNodeMatrixEntry(BaseModel): eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) +class SingleNodeFixedArMtpMatrixEntry(SingleNodeMatrixEntry): + """Single-node throughput entry with synthetic fixed acceptance rates for MTP.""" + model_config = ConfigDict(extra='forbid', populate_by_name=True) + + scenario_type: Literal["fixed-ar-mtp"] = Field(alias=Fields.SCENARIO_TYPE.value) + draft_model: str = Field(alias=Fields.DRAFT_MODEL.value) + num_speculative_tokens: int = Field(alias=Fields.NUM_SPECULATIVE_TOKENS.value) + rejection_sample_method: Literal["synthetic"] = Field(alias=Fields.REJECTION_SAMPLE_METHOD.value) + synthetic_acceptance_rates: List[float] = Field(alias=Fields.SYNTHETIC_ACCEPTANCE_RATES.value) + + @model_validator(mode='after') + def validate_synthetic_acceptance_rates(self): + if self.spec_decoding != "eagle3": + raise ValueError("fixed-ar-mtp entries must use spec-decoding: eagle3") + if len(self.synthetic_acceptance_rates) != self.num_speculative_tokens: + raise ValueError( + f"'{Fields.SYNTHETIC_ACCEPTANCE_RATES.value}' length must match " + f"'{Fields.NUM_SPECULATIVE_TOKENS.value}'" + ) + return self + + class WorkerConfig(BaseModel): """Pydantic model for validating worker configuration in multinode entries.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) @@ -125,7 +154,7 @@ class MultiNodeMatrixEntry(BaseModel): model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) precision: str framework: str - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "eagle3", "draft_model", "none"] = Field( alias=Fields.SPEC_DECODING.value ) runner: str @@ -171,7 +200,7 @@ class MultiNodeAgenticMatrixEntry(BaseModel): model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) precision: str framework: str - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "eagle3", "draft_model", "none"] = Field( alias=Fields.SPEC_DECODING.value ) runner: str @@ -207,7 +236,11 @@ def validate_matrix_entry(entry: dict, is_multinode: bool) -> dict: Returns the original list if all entries are valid. """ try: - if is_multinode: + if entry.get(Fields.SCENARIO_TYPE.value) == Fields.FIXED_AR_MTP.value: + if is_multinode: + raise ValueError("fixed-ar-mtp is only supported for single-node entries") + SingleNodeFixedArMtpMatrixEntry(**entry) + elif is_multinode: MultiNodeMatrixEntry(**entry) else: SingleNodeMatrixEntry(**entry) @@ -271,7 +304,7 @@ class SingleNodeSearchSpaceEntry(BaseModel): tp: int ep: Optional[int] = None - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "eagle3", "draft_model", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value) dp_attn: Optional[bool] = Field( default=None, alias=Fields.DP_ATTN.value) @@ -291,7 +324,7 @@ class MultiNodeSearchSpaceEntry(BaseModel): """Multinode search space configuration.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "eagle3", "draft_model", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value) prefill: WorkerConfig decode: WorkerConfig @@ -317,6 +350,28 @@ class SingleNodeSeqLenConfig(BaseModel): alias=Fields.SEARCH_SPACE.value) +class FixedArMtpSeqLenConfig(BaseModel): + """Single-node fixed-AR MTP scenario configuration.""" + model_config = ConfigDict(extra='forbid', populate_by_name=True) + + isl: int + osl: int + draft_model: str = Field(alias=Fields.DRAFT_MODEL.value) + num_speculative_tokens: int = Field(alias=Fields.NUM_SPECULATIVE_TOKENS.value) + rejection_sample_method: Literal["synthetic"] = Field(alias=Fields.REJECTION_SAMPLE_METHOD.value) + synthetic_acceptance_rates: List[float] = Field(alias=Fields.SYNTHETIC_ACCEPTANCE_RATES.value) + search_space: List[SingleNodeSearchSpaceEntry] = Field(alias=Fields.SEARCH_SPACE.value) + + @model_validator(mode='after') + def validate_synthetic_acceptance_rates(self): + if len(self.synthetic_acceptance_rates) != self.num_speculative_tokens: + raise ValueError( + f"'{Fields.SYNTHETIC_ACCEPTANCE_RATES.value}' length must match " + f"'{Fields.NUM_SPECULATIVE_TOKENS.value}'" + ) + return self + + class MultiNodeSeqLenConfig(BaseModel): """Multinode sequence length configuration.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) @@ -334,7 +389,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel): tp: Optional[int] = None ep: Optional[int] = None dp_attn: Optional[bool] = Field(default=None, alias=Fields.DP_ATTN.value) - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "eagle3", "draft_model", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value) prefill: Optional[WorkerConfig] = None decode: Optional[WorkerConfig] = None @@ -375,12 +430,14 @@ class SingleNodeScenarios(BaseModel): fixed_seq_len: Optional[List[SingleNodeSeqLenConfig]] = Field( default=None, alias=Fields.FIXED_SEQ_LEN.value) + fixed_ar_mtp: Optional[List[FixedArMtpSeqLenConfig]] = Field( + default=None, alias=Fields.FIXED_AR_MTP.value) agentic_coding: Optional[List[AgenticCodingConfig]] = Field( default=None, alias=Fields.AGENTIC_CODING.value) @model_validator(mode='after') def at_least_one_scenario(self): - if not self.fixed_seq_len and not self.agentic_coding: + if not self.fixed_seq_len and not self.fixed_ar_mtp and not self.agentic_coding: raise ValueError("At least one scenario type must be specified") return self @@ -504,7 +561,7 @@ class ChangelogMatrixEntry(BaseModel): """ model_config = ConfigDict(extra="forbid", populate_by_name=True) - single_node: dict[str, list[Union[SingleNodeMatrixEntry, SingleNodeAgenticMatrixEntry]] + single_node: dict[str, list[Union[SingleNodeMatrixEntry, SingleNodeFixedArMtpMatrixEntry, SingleNodeAgenticMatrixEntry]] ] = Field(default_factory=dict) multi_node: dict[str, list[Union[MultiNodeMatrixEntry, MultiNodeAgenticMatrixEntry]] ] = Field(default_factory=dict)