From 017f66e9a86c269f4bdfb1c490ae0cdb5740707a Mon Sep 17 00:00:00 2001 From: Youhezhen Date: Wed, 22 Jul 2026 18:05:50 -0700 Subject: [PATCH] Update: make dense dependency diagnostics debug-only - Keep fanin and fanout threshold checks but emit them at DEBUG level for both a2a3 and a5 runtimes. - Exercise high-degree fanin and fanout wiring in dummy-task scenes. This keeps the diagnostic available on demand without adding noise to normal runtime logs. --- .../runtime/pto_orchestrator.cpp | 26 +++++++++---------- .../runtime/pto_runtime2_types.h | 8 +++--- .../runtime/pto_orchestrator.cpp | 26 +++++++++---------- .../runtime/pto_runtime2_types.h | 8 +++--- .../kernels/orchestration/dummy_task_orch.cpp | 11 +++----- .../dummy_task/test_dummy_task.py | 13 ++++++---- .../kernels/orchestration/dummy_task_orch.cpp | 11 +++----- .../dummy_task/test_dummy_task.py | 13 ++++++---- 8 files changed, 56 insertions(+), 60 deletions(-) diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp index c4e6464b46..d5f4bfcd22 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp @@ -318,14 +318,13 @@ static bool append_fanin_or_fail( // lock + unlock always; one fanout_count store when we actually claim. g_orch_args_atomic_count += claim ? 3 : 2; #endif - // Dense-fanout diagnostic, emitted outside fanout_lock (no logging under the - // spinlock). The monotonic fanout_count++ crosses THRESHOLD+1 exactly once, - // so each dense producer warns exactly once and the AICPU hot path keeps a - // single predicted-not-taken branch on the common case. - if (fanout_now == PTO2_DEP_DEGREE_WARN_THRESHOLD + 1) { - LOG_WARN( + // Dense-fanout diagnostic, emitted outside fanout_lock (no logging under + // the spinlock). The monotonic fanout_count++ crosses THRESHOLD+1 exactly + // once, so each dense producer emits one debug message when enabled. + if (fanout_now == PTO2_DEP_DEGREE_DEBUG_THRESHOLD + 1) { + LOG_DEBUG( "dense dependency: task ring=%u id=%u fanout>%d [orch submit]", - static_cast(producer_task_id.ring()), producer_task_id.local(), PTO2_DEP_DEGREE_WARN_THRESHOLD + static_cast(producer_task_id.ring()), producer_task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD ); } // gone (stale/consumed) or an already-seen duplicate live producer: no new @@ -947,14 +946,13 @@ static TaskOutputTensors submit_task_common( int32_t inline_count = std::min(fanin_builder.count, PTO2_FANIN_INLINE_CAP); // Store fanin metadata in payload for scheduler to iterate payload.fanin_actual_count = fanin_builder.count; - // Dense-fanin diagnostic: fanin_builder.count is finalized here and submit - // runs once per task, so each dense consumer warns exactly once. The check - // is > THRESHOLD (not == THRESHOLD+1): the count lands at its total here, so - // an == crossing test would miss a task that jumps straight past THRESHOLD+1. - if (fanin_builder.count > PTO2_DEP_DEGREE_WARN_THRESHOLD) { - LOG_WARN( + // fanin_builder.count is finalized here and submit runs once per task, so + // each dense consumer emits one debug message when enabled. This checks > + // THRESHOLD because the count lands at its final total here. + if (fanin_builder.count > PTO2_DEP_DEGREE_DEBUG_THRESHOLD) { + LOG_DEBUG( "dense dependency: task ring=%u id=%u fanin>%d [orch submit]", static_cast(task_id.ring()), - task_id.local(), PTO2_DEP_DEGREE_WARN_THRESHOLD + task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD ); } payload.fanin_spill_start = fanin_builder.spill_start; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h index 8645a0b7f8..d10dc107b6 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h @@ -93,10 +93,10 @@ // Fanin storage #define PTO2_FANIN_INLINE_CAP 64 -// Dependency-degree diagnostic: warn once when a task's fanin or a producer's -// fanout first exceeds this degree, so dense dependency graphs surface without -// flooding the AICPU hot-path device log. -#define PTO2_DEP_DEGREE_WARN_THRESHOLD 16 +// Dependency-degree diagnostic: log once at debug level when a task's fanin or +// a producer's fanout first exceeds this degree, so dense dependency graphs can +// be inspected without adding noise to normal runtime logs. +#define PTO2_DEP_DEGREE_DEBUG_THRESHOLD 16 // TensorMap cleanup interval #define PTO2_TENSORMAP_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp index fa352aedec..9d6dd44b61 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp @@ -312,14 +312,13 @@ static bool append_fanin_or_fail( // lock + unlock always; one fanout_count store when we actually claim. g_orch_args_atomic_count += claim ? 3 : 2; #endif - // Dense-fanout diagnostic, emitted outside fanout_lock (no logging under the - // spinlock). The monotonic fanout_count++ crosses THRESHOLD+1 exactly once, - // so each dense producer warns exactly once and the AICPU hot path keeps a - // single predicted-not-taken branch on the common case. - if (fanout_now == PTO2_DEP_DEGREE_WARN_THRESHOLD + 1) { - LOG_WARN( + // Dense-fanout diagnostic, emitted outside fanout_lock (no logging under + // the spinlock). The monotonic fanout_count++ crosses THRESHOLD+1 exactly + // once, so each dense producer emits one debug message when enabled. + if (fanout_now == PTO2_DEP_DEGREE_DEBUG_THRESHOLD + 1) { + LOG_DEBUG( "dense dependency: task ring=%u id=%u fanout>%d [orch submit]", - static_cast(producer_task_id.ring()), producer_task_id.local(), PTO2_DEP_DEGREE_WARN_THRESHOLD + static_cast(producer_task_id.ring()), producer_task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD ); } // gone (stale/consumed) or an already-seen duplicate live producer: no new @@ -949,14 +948,13 @@ static TaskOutputTensors submit_task_common( int32_t inline_count = std::min(fanin_builder.count, PTO2_FANIN_INLINE_CAP); // Store fanin metadata in payload for scheduler to iterate payload.fanin_actual_count = fanin_builder.count; - // Dense-fanin diagnostic: fanin_builder.count is finalized here and submit - // runs once per task, so each dense consumer warns exactly once. The check - // is > THRESHOLD (not == THRESHOLD+1): the count lands at its total here, so - // an == crossing test would miss a task that jumps straight past THRESHOLD+1. - if (fanin_builder.count > PTO2_DEP_DEGREE_WARN_THRESHOLD) { - LOG_WARN( + // fanin_builder.count is finalized here and submit runs once per task, so + // each dense consumer emits one debug message when enabled. This checks > + // THRESHOLD because the count lands at its final total here. + if (fanin_builder.count > PTO2_DEP_DEGREE_DEBUG_THRESHOLD) { + LOG_DEBUG( "dense dependency: task ring=%u id=%u fanin>%d [orch submit]", static_cast(task_id.ring()), - task_id.local(), PTO2_DEP_DEGREE_WARN_THRESHOLD + task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD ); } payload.fanin_spill_start = fanin_builder.spill_start; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h index 5c7de9199d..e70a680c0a 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h @@ -91,10 +91,10 @@ // Fanin storage #define PTO2_FANIN_INLINE_CAP 64 -// Dependency-degree diagnostic: warn once when a task's fanin or a producer's -// fanout first exceeds this degree, so dense dependency graphs surface without -// flooding the AICPU hot-path device log. -#define PTO2_DEP_DEGREE_WARN_THRESHOLD 16 +// Dependency-degree diagnostic: log once at debug level when a task's fanin or +// a producer's fanout first exceeds this degree, so dense dependency graphs can +// be inspected without adding noise to normal runtime logs. +#define PTO2_DEP_DEGREE_DEBUG_THRESHOLD 16 // TensorMap cleanup interval #define PTO2_TENSORMAP_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp b/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp index 1860163570..f041c5523d 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp @@ -49,8 +49,8 @@ #define FUNC_COPY_FIRST 1 static constexpr int32_t LONG_CHAIN_DUMMIES = 4; -// case=4 dense-dependency degree: > PTO2_DEP_DEGREE_WARN_THRESHOLD (16) so the -// producer's fanout and the final consumer's fanin both trip the diagnostic. +// case=4 exceeds PTO2_DEP_DEGREE_DEBUG_THRESHOLD (16), exercising both the +// producer-fanout and final-consumer-fanin debug diagnostics. static constexpr int32_t DENSE_DEP_COUNT = 18; extern "C" { @@ -142,11 +142,8 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const L2Ta rt_submit_aic_task(FUNC_COPY_FIRST, args); } } else if (case_id == 4) { - // Dense fanout + fanin: one producer feeds DENSE_DEP_COUNT dummy - // barriers (producer fanout = DENSE_DEP_COUNT), then one consumer - // depends on all of them (consumer fanin = DENSE_DEP_COUNT). Both - // degrees exceed PTO2_DEP_DEGREE_WARN_THRESHOLD, tripping the - // orchestrator's dense-fanout and dense-fanin diagnostics. + // One producer feeds DENSE_DEP_COUNT dummy barriers, then one consumer + // depends on all of them, exercising both dense-dependency diagnostics. PTO2TaskId a_id; { L0TaskArgs args; diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py b/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py index 2b88d24991..7d75488d77 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py @@ -9,7 +9,7 @@ # ----------------------------------------------------------------------------------------------------------- """dummy_task: verify dep-only tasks block consumers and never run a kernel. -The orchestration submits one of three scenes, controlled by params["case"]: +The orchestration submits one of four scenes, controlled by params["case"]: case=1 (single dummy via tensormap INOUT): producer writes X[0]=42.0 -> dummy_T INOUTs X -> consumer copies X to Y. @@ -27,6 +27,10 @@ set_dependencies({A, B}, 2) as a many-to-one barrier; the consumer set_dependencies({dummy}, 1) and reads X. Verifies dummy_task participates in explicit_dep wiring. + + case=4 (dense fanout and fanin): + One producer feeds 18 dummy barriers, then one consumer depends on all + 18. Verifies high-degree dummy-task dependency wiring. """ import torch @@ -91,10 +95,9 @@ class TestDummyTask(SceneTestCase): "params": {"case": 3}, }, { - # One producer fanned out to 18 dummy barriers, then one consumer - # depending on all 18 — both degrees exceed the dense-dependency - # warn threshold (16), exercising the orchestrator's fanout and - # fanin diagnostics. Correctness is still just the copy. + # One producer fans out to 18 dummy barriers, then one consumer + # depends on all 18, exercising the dense-dependency debug paths. + # Correctness is still just the copy. "name": "DenseFanoutFanin", "platforms": ["a2a3sim", "a2a3"], "config": {"aicpu_thread_num": 2, "block_dim": 1}, diff --git a/tests/st/a5/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp b/tests/st/a5/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp index 1860163570..f041c5523d 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp +++ b/tests/st/a5/tensormap_and_ringbuffer/dummy_task/kernels/orchestration/dummy_task_orch.cpp @@ -49,8 +49,8 @@ #define FUNC_COPY_FIRST 1 static constexpr int32_t LONG_CHAIN_DUMMIES = 4; -// case=4 dense-dependency degree: > PTO2_DEP_DEGREE_WARN_THRESHOLD (16) so the -// producer's fanout and the final consumer's fanin both trip the diagnostic. +// case=4 exceeds PTO2_DEP_DEGREE_DEBUG_THRESHOLD (16), exercising both the +// producer-fanout and final-consumer-fanin debug diagnostics. static constexpr int32_t DENSE_DEP_COUNT = 18; extern "C" { @@ -142,11 +142,8 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const L2Ta rt_submit_aic_task(FUNC_COPY_FIRST, args); } } else if (case_id == 4) { - // Dense fanout + fanin: one producer feeds DENSE_DEP_COUNT dummy - // barriers (producer fanout = DENSE_DEP_COUNT), then one consumer - // depends on all of them (consumer fanin = DENSE_DEP_COUNT). Both - // degrees exceed PTO2_DEP_DEGREE_WARN_THRESHOLD, tripping the - // orchestrator's dense-fanout and dense-fanin diagnostics. + // One producer feeds DENSE_DEP_COUNT dummy barriers, then one consumer + // depends on all of them, exercising both dense-dependency diagnostics. PTO2TaskId a_id; { L0TaskArgs args; diff --git a/tests/st/a5/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py b/tests/st/a5/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py index aa5014443c..268055ad2d 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py @@ -9,7 +9,7 @@ # ----------------------------------------------------------------------------------------------------------- """dummy_task: verify dep-only tasks block consumers and never run a kernel. -The orchestration submits one of three scenes, controlled by params["case"]: +The orchestration submits one of four scenes, controlled by params["case"]: case=1 (single dummy via tensormap INOUT): producer writes X[0]=42.0 -> dummy_T INOUTs X -> consumer copies X to Y. @@ -27,6 +27,10 @@ set_dependencies({A, B}, 2) as a many-to-one barrier; the consumer set_dependencies({dummy}, 1) and reads X. Verifies dummy_task participates in explicit_dep wiring. + + case=4 (dense fanout and fanin): + One producer feeds 18 dummy barriers, then one consumer depends on all + 18. Verifies high-degree dummy-task dependency wiring. """ import torch @@ -87,10 +91,9 @@ class TestDummyTask(SceneTestCase): "params": {"case": 3}, }, { - # One producer fanned out to 18 dummy barriers, then one consumer - # depending on all 18 — both degrees exceed the dense-dependency - # warn threshold (16), exercising the orchestrator's fanout and - # fanin diagnostics. Correctness is still just the copy. + # One producer fans out to 18 dummy barriers, then one consumer + # depends on all 18, exercising the dense-dependency debug paths. + # Correctness is still just the copy. "name": "DenseFanoutFanin", "platforms": ["a5sim", "a5"], "config": {"aicpu_thread_num": 2, "block_dim": 1},