[CI Failure]: Simple · Unit Test, RuntimeError: CAMAsyncAFDConnector attention DPxTP world size must equal its configured role size, got 2 and 1

[yenuo26]

Failing job or test

tests/unit/v1/worker/test_npu_runtime.py::test_npu_async_moe_ubatching_validation_accepts_supported_shape[token-attention-tp2]

Basic information

• Flaky test
• Reproduces locally
• CPU-only failure
• GPU-gated failure
• Related to optional vLLM dependency

Failure details

https://buildkite.com/vllm/afd-plugin/builds/86/canvas?sid=01a071d8-f1db-4722-a238-b09fd2a99c64&tab=output

_____________________________________ test_npu_async_moe_ubatching_validation_accepts_supported_shape[token-attention-tp2] _____________________________________
config = namespace(additional_config={'afd': {'role': 'attention', 'connector': 'CAMAsyncAFDConnector', 'async': True, 'compute...aphs=<function _vllm_config.<locals>.<lambda> at 0x7fc14e72f4c0>), fast_moe_cold_start=False), speculative_config=None)

@pytest.mark.parametrize(
    "config",
    [
        pytest.param(_async_moe_config(), id="request-tp1"),
        pytest.param(
            _async_moe_config(
                tensor_parallel_size=2,
                async_moe_split="token",
            ),
            id="token-attention-tp2",
        ),
        pytest.param(
            _async_moe_config(role="ffn", async_moe_split="token"),
            id="token-ffn-tp1",
        ),
    ],
)
def test_npu_async_moe_ubatching_validation_accepts_supported_shape(config):
>       fail_if_unsupported_npu_afd_features(config)

tests/unit/v1/worker/test_npu_runtime.py:2285:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
afd_plugin/compat/npu/feature_validation.py:42: in fail_if_unsupported_npu_afd_features
    _fail_if_unsupported_npu_afd_async_features(
afd_plugin/compat/npu/feature_validation.py:176: in _fail_if_unsupported_npu_afd_async_features
    _validate_cam_world_topology(vllm_config, afd_config, extra_info)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

vllm_config = namespace(additional_config={'afd': {'role': 'attention', 'connector': 'CAMAsyncAFDConnector', 'async': True, 'compute...aphs=<function _vllm_config.<locals>.<lambda> at 0x7fc14e72f4c0>), fast_moe_cold_start=False), speculative_config=None)
afd_config = AFDConfig(connector='CAMAsyncAFDConnector', async_dp=True, role='attention', port=1239, host='127.0.0.1', num_attention_ranks=1, num_ffn_ranks=1, compute_gate_on_attention=True)
extra_info = AFDAsyncExtraInfo(dynamic_quant=0, attn_ranks_per_dp=2, async_moe_ubatching=True, async_moe_num_ubatches=2, async_moe_split='token', hccl_buffer_size=None)

def _validate_cam_world_topology(
    vllm_config: VllmConfig,
    afd_config: AFDConfig,
    extra_info: ConnectorExtraInfo,
) -> None:
    """Require each role's local layout to fill the one CAM world."""
    from afd_plugin.connectors.npu.async_cam import AFDAsyncExtraInfo
    if not isinstance(extra_info, AFDAsyncExtraInfo):
        return
    parallel_config = vllm_config.parallel_config
    attn_ranks_per_dp = int(extra_info.attn_ranks_per_dp)
    if afd_config.role == "attention":
        if int(parallel_config.tensor_parallel_size) != attn_ranks_per_dp:
            raise RuntimeError(
                "CAMAsyncAFDConnector Attention tensor_parallel_size must equal "
                "attn_ranks_per_dp",
            )
        local_world_size = int(parallel_config.data_parallel_size) * attn_ranks_per_dp
        expected_world_size = afd_config.num_attention_ranks
    else:
        local_world_size = int(parallel_config.data_parallel_size) * int(
            parallel_config.tensor_parallel_size
        )
        expected_world_size = afd_config.num_ffn_ranks
    if local_world_size != expected_world_size:
>       raise RuntimeError(
            "CAMAsyncAFDConnector "
            f"{afd_config.role} DPxTP world size must equal its configured "
            f"role size, got {local_world_size} and {expected_world_size}",
        )

E       RuntimeError: CAMAsyncAFDConnector attention DPxTP world size must equal its configured role size, got 2 and 1

afd_plugin/compat/npu/feature_validation.py:205: RuntimeError

Failure history

No response

Local reproduction

CC list

No response

[yenuo26]

@jiangkuaixue123 After 08c2ded merged, this test fail. PTAL