Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -318,14 +318,13 @@ static bool append_fanin_or_fail(
// lock + unlock always; one fanout_count store when we actually claim.
g_orch_args_atomic_count += claim ? 3 : 2;
#endif
// Dense-fanout diagnostic, emitted outside fanout_lock (no logging under the
// spinlock). The monotonic fanout_count++ crosses THRESHOLD+1 exactly once,
// so each dense producer warns exactly once and the AICPU hot path keeps a
// single predicted-not-taken branch on the common case.
if (fanout_now == PTO2_DEP_DEGREE_WARN_THRESHOLD + 1) {
LOG_WARN(
// Dense-fanout diagnostic, emitted outside fanout_lock (no logging under
// the spinlock). The monotonic fanout_count++ crosses THRESHOLD+1 exactly
// once, so each dense producer emits one debug message when enabled.
if (fanout_now == PTO2_DEP_DEGREE_DEBUG_THRESHOLD + 1) {
LOG_DEBUG(
"dense dependency: task ring=%u id=%u fanout>%d [orch submit]",
static_cast<unsigned>(producer_task_id.ring()), producer_task_id.local(), PTO2_DEP_DEGREE_WARN_THRESHOLD
static_cast<unsigned>(producer_task_id.ring()), producer_task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD
);
}
// gone (stale/consumed) or an already-seen duplicate live producer: no new
Expand Down Expand Up @@ -947,14 +946,13 @@ static TaskOutputTensors submit_task_common(
int32_t inline_count = std::min(fanin_builder.count, PTO2_FANIN_INLINE_CAP);
// Store fanin metadata in payload for scheduler to iterate
payload.fanin_actual_count = fanin_builder.count;
// Dense-fanin diagnostic: fanin_builder.count is finalized here and submit
// runs once per task, so each dense consumer warns exactly once. The check
// is > THRESHOLD (not == THRESHOLD+1): the count lands at its total here, so
// an == crossing test would miss a task that jumps straight past THRESHOLD+1.
if (fanin_builder.count > PTO2_DEP_DEGREE_WARN_THRESHOLD) {
LOG_WARN(
// fanin_builder.count is finalized here and submit runs once per task, so
// each dense consumer emits one debug message when enabled. This checks >
// THRESHOLD because the count lands at its final total here.
if (fanin_builder.count > PTO2_DEP_DEGREE_DEBUG_THRESHOLD) {
LOG_DEBUG(
"dense dependency: task ring=%u id=%u fanin>%d [orch submit]", static_cast<unsigned>(task_id.ring()),
task_id.local(), PTO2_DEP_DEGREE_WARN_THRESHOLD
task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD
);
}
payload.fanin_spill_start = fanin_builder.spill_start;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,10 @@
// Fanin storage
#define PTO2_FANIN_INLINE_CAP 64

// Dependency-degree diagnostic: warn once when a task's fanin or a producer's
// fanout first exceeds this degree, so dense dependency graphs surface without
// flooding the AICPU hot-path device log.
#define PTO2_DEP_DEGREE_WARN_THRESHOLD 16
// Dependency-degree diagnostic: log once at debug level when a task's fanin or
// a producer's fanout first exceeds this degree, so dense dependency graphs can
// be inspected without adding noise to normal runtime logs.
#define PTO2_DEP_DEGREE_DEBUG_THRESHOLD 16

// TensorMap cleanup interval
#define PTO2_TENSORMAP_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,14 +312,13 @@ static bool append_fanin_or_fail(
// lock + unlock always; one fanout_count store when we actually claim.
g_orch_args_atomic_count += claim ? 3 : 2;
#endif
// Dense-fanout diagnostic, emitted outside fanout_lock (no logging under the
// spinlock). The monotonic fanout_count++ crosses THRESHOLD+1 exactly once,
// so each dense producer warns exactly once and the AICPU hot path keeps a
// single predicted-not-taken branch on the common case.
if (fanout_now == PTO2_DEP_DEGREE_WARN_THRESHOLD + 1) {
LOG_WARN(
// Dense-fanout diagnostic, emitted outside fanout_lock (no logging under
// the spinlock). The monotonic fanout_count++ crosses THRESHOLD+1 exactly
// once, so each dense producer emits one debug message when enabled.
if (fanout_now == PTO2_DEP_DEGREE_DEBUG_THRESHOLD + 1) {
LOG_DEBUG(
"dense dependency: task ring=%u id=%u fanout>%d [orch submit]",
static_cast<unsigned>(producer_task_id.ring()), producer_task_id.local(), PTO2_DEP_DEGREE_WARN_THRESHOLD
static_cast<unsigned>(producer_task_id.ring()), producer_task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD
);
}
// gone (stale/consumed) or an already-seen duplicate live producer: no new
Expand Down Expand Up @@ -949,14 +948,13 @@ static TaskOutputTensors submit_task_common(
int32_t inline_count = std::min(fanin_builder.count, PTO2_FANIN_INLINE_CAP);
// Store fanin metadata in payload for scheduler to iterate
payload.fanin_actual_count = fanin_builder.count;
// Dense-fanin diagnostic: fanin_builder.count is finalized here and submit
// runs once per task, so each dense consumer warns exactly once. The check
// is > THRESHOLD (not == THRESHOLD+1): the count lands at its total here, so
// an == crossing test would miss a task that jumps straight past THRESHOLD+1.
if (fanin_builder.count > PTO2_DEP_DEGREE_WARN_THRESHOLD) {
LOG_WARN(
// fanin_builder.count is finalized here and submit runs once per task, so
// each dense consumer emits one debug message when enabled. This checks >
// THRESHOLD because the count lands at its final total here.
if (fanin_builder.count > PTO2_DEP_DEGREE_DEBUG_THRESHOLD) {
LOG_DEBUG(
"dense dependency: task ring=%u id=%u fanin>%d [orch submit]", static_cast<unsigned>(task_id.ring()),
task_id.local(), PTO2_DEP_DEGREE_WARN_THRESHOLD
task_id.local(), PTO2_DEP_DEGREE_DEBUG_THRESHOLD
);
}
payload.fanin_spill_start = fanin_builder.spill_start;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@
// Fanin storage
#define PTO2_FANIN_INLINE_CAP 64

// Dependency-degree diagnostic: warn once when a task's fanin or a producer's
// fanout first exceeds this degree, so dense dependency graphs surface without
// flooding the AICPU hot-path device log.
#define PTO2_DEP_DEGREE_WARN_THRESHOLD 16
// Dependency-degree diagnostic: log once at debug level when a task's fanin or
// a producer's fanout first exceeds this degree, so dense dependency graphs can
// be inspected without adding noise to normal runtime logs.
#define PTO2_DEP_DEGREE_DEBUG_THRESHOLD 16

// TensorMap cleanup interval
#define PTO2_TENSORMAP_CLEANUP_INTERVAL 64 // Cleanup every N retired tasks
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@
#define FUNC_COPY_FIRST 1

static constexpr int32_t LONG_CHAIN_DUMMIES = 4;
// case=4 dense-dependency degree: > PTO2_DEP_DEGREE_WARN_THRESHOLD (16) so the
// producer's fanout and the final consumer's fanin both trip the diagnostic.
// case=4 exceeds PTO2_DEP_DEGREE_DEBUG_THRESHOLD (16), exercising both the
// producer-fanout and final-consumer-fanin debug diagnostics.
static constexpr int32_t DENSE_DEP_COUNT = 18;

extern "C" {
Expand Down Expand Up @@ -142,11 +142,8 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const L2Ta
rt_submit_aic_task(FUNC_COPY_FIRST, args);
}
} else if (case_id == 4) {
// Dense fanout + fanin: one producer feeds DENSE_DEP_COUNT dummy
// barriers (producer fanout = DENSE_DEP_COUNT), then one consumer
// depends on all of them (consumer fanin = DENSE_DEP_COUNT). Both
// degrees exceed PTO2_DEP_DEGREE_WARN_THRESHOLD, tripping the
// orchestrator's dense-fanout and dense-fanin diagnostics.
// One producer feeds DENSE_DEP_COUNT dummy barriers, then one consumer
// depends on all of them, exercising both dense-dependency diagnostics.
PTO2TaskId a_id;
{
L0TaskArgs args;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
# -----------------------------------------------------------------------------------------------------------
"""dummy_task: verify dep-only tasks block consumers and never run a kernel.

The orchestration submits one of three scenes, controlled by params["case"]:
The orchestration submits one of four scenes, controlled by params["case"]:

case=1 (single dummy via tensormap INOUT):
producer writes X[0]=42.0 -> dummy_T INOUTs X -> consumer copies X to Y.
Expand All @@ -27,6 +27,10 @@
set_dependencies({A, B}, 2) as a many-to-one barrier; the consumer
set_dependencies({dummy}, 1) and reads X. Verifies dummy_task
participates in explicit_dep wiring.

case=4 (dense fanout and fanin):
One producer feeds 18 dummy barriers, then one consumer depends on all
18. Verifies high-degree dummy-task dependency wiring.
"""

import torch
Expand Down Expand Up @@ -91,10 +95,9 @@ class TestDummyTask(SceneTestCase):
"params": {"case": 3},
},
{
# One producer fanned out to 18 dummy barriers, then one consumer
# depending on all 18 — both degrees exceed the dense-dependency
# warn threshold (16), exercising the orchestrator's fanout and
# fanin diagnostics. Correctness is still just the copy.
# One producer fans out to 18 dummy barriers, then one consumer
# depends on all 18, exercising the dense-dependency debug paths.
# Correctness is still just the copy.
"name": "DenseFanoutFanin",
"platforms": ["a2a3sim", "a2a3"],
"config": {"aicpu_thread_num": 2, "block_dim": 1},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@
#define FUNC_COPY_FIRST 1

static constexpr int32_t LONG_CHAIN_DUMMIES = 4;
// case=4 dense-dependency degree: > PTO2_DEP_DEGREE_WARN_THRESHOLD (16) so the
// producer's fanout and the final consumer's fanin both trip the diagnostic.
// case=4 exceeds PTO2_DEP_DEGREE_DEBUG_THRESHOLD (16), exercising both the
// producer-fanout and final-consumer-fanin debug diagnostics.
static constexpr int32_t DENSE_DEP_COUNT = 18;

extern "C" {
Expand Down Expand Up @@ -142,11 +142,8 @@ __attribute__((visibility("default"))) void aicpu_orchestration_entry(const L2Ta
rt_submit_aic_task(FUNC_COPY_FIRST, args);
}
} else if (case_id == 4) {
// Dense fanout + fanin: one producer feeds DENSE_DEP_COUNT dummy
// barriers (producer fanout = DENSE_DEP_COUNT), then one consumer
// depends on all of them (consumer fanin = DENSE_DEP_COUNT). Both
// degrees exceed PTO2_DEP_DEGREE_WARN_THRESHOLD, tripping the
// orchestrator's dense-fanout and dense-fanin diagnostics.
// One producer feeds DENSE_DEP_COUNT dummy barriers, then one consumer
// depends on all of them, exercising both dense-dependency diagnostics.
PTO2TaskId a_id;
{
L0TaskArgs args;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
# -----------------------------------------------------------------------------------------------------------
"""dummy_task: verify dep-only tasks block consumers and never run a kernel.

The orchestration submits one of three scenes, controlled by params["case"]:
The orchestration submits one of four scenes, controlled by params["case"]:

case=1 (single dummy via tensormap INOUT):
producer writes X[0]=42.0 -> dummy_T INOUTs X -> consumer copies X to Y.
Expand All @@ -27,6 +27,10 @@
set_dependencies({A, B}, 2) as a many-to-one barrier; the consumer
set_dependencies({dummy}, 1) and reads X. Verifies dummy_task
participates in explicit_dep wiring.

case=4 (dense fanout and fanin):
One producer feeds 18 dummy barriers, then one consumer depends on all
18. Verifies high-degree dummy-task dependency wiring.
"""

import torch
Expand Down Expand Up @@ -87,10 +91,9 @@ class TestDummyTask(SceneTestCase):
"params": {"case": 3},
},
{
# One producer fanned out to 18 dummy barriers, then one consumer
# depending on all 18 — both degrees exceed the dense-dependency
# warn threshold (16), exercising the orchestrator's fanout and
# fanin diagnostics. Correctness is still just the copy.
# One producer fans out to 18 dummy barriers, then one consumer
# depends on all 18, exercising the dense-dependency debug paths.
# Correctness is still just the copy.
"name": "DenseFanoutFanin",
"platforms": ["a5sim", "a5"],
"config": {"aicpu_thread_num": 2, "block_dim": 1},
Expand Down
Loading