diff --git a/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp b/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp index 1b966347e..2023096b5 100644 --- a/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp +++ b/src/a2a3/runtime/host_build_graph/aicpu/aicpu_executor.cpp @@ -288,7 +288,11 @@ int32_t AicpuExecutor::run(Runtime *runtime) { LOG_ERROR("Thread %d: rt is null after orchestrator error, skipping dispatch", thread_idx); } else { sched_ctx_.bind_runtime(rt); - int32_t completed = sched_ctx_.resolve_and_dispatch(runtime, thread_idx); + // 3S+1P: the last thread is the core-less resolution (P) thread; the + // rest are core-owning schedulers (S). + int32_t completed = (thread_idx == sched_ctx_.p_thread_idx()) ? + sched_ctx_.run_resolution_thread(runtime, thread_idx) : + sched_ctx_.resolve_and_dispatch(runtime, thread_idx); if (completed < 0) { LOG_ERROR("Thread %d: Scheduler failed with rc=%d", thread_idx, completed); run_rc = completed; diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp index 3cc67f03b..1fcc13e71 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_cold_path.cpp @@ -700,7 +700,21 @@ void SchedulerContext::handshake_partition(Runtime *runtime, int32_t tidx, int32 bool SchedulerContext::assign_cores_to_threads() { // Cluster-aligned round-robin assignment: cluster ci -> sched thread ci % active_sched_threads_. // Each cluster = 1 AIC + 2 adjacent AIV; the triple is always kept together. - active_sched_threads_ = aicpu_thread_num_; + // + // 3S+1P: the last AICPU thread is the core-less resolution thread (P); cores + // partition across the remaining (aicpu_thread_num_ - 1) scheduler threads + // only, so P never owns a cluster and never polls a COND register. P is + // mandatory — like tmr's scheduler + orchestrator split, host_build_graph + // needs at least two AICPU threads (one S + one P); one thread cannot own + // cores and resolve on a dedicated thread at once. + if (aicpu_thread_num_ < 2) { + LOG_ERROR( + "host_build_graph requires aicpu_thread_num >= 2 (1 scheduler + 1 resolution); got %d", aicpu_thread_num_ + ); + return false; + } + p_thread_idx_ = aicpu_thread_num_ - 1; + active_sched_threads_ = aicpu_thread_num_ - 1; int32_t cluster_count = aic_count_; // Max clusters any single sched thread can hold: ceil(cluster_count / active_sched_threads_). @@ -1040,6 +1054,29 @@ void SchedulerContext::on_orchestration_done( total_tasks_ = total_tasks; + // Allocate the per-S CompletedTaskQueues here on the boot leader, before it + // releases runtime_init_ready_ — no scheduler thread can push until then. + // Completed-but-unresolved tasks in flight are bounded by BOTH the total task + // count and the ring's task window (a task must occupy a ring slot to run and + // complete), so size to the tighter of the two, rounded up to a power of two + // and floored at 256. The window already caps this, so there is no artificial + // ceiling and a producer never has to spin on a full queue. + uint64_t sp_bound = static_cast(total_tasks); + if (sched_->ring_sched_state.ring != nullptr) { + uint64_t window = static_cast(sched_->ring_sched_state.ring->task_window_mask) + 1; + if (window < sp_bound) { + sp_bound = window; + } + } + uint64_t sp_cap = 256; + while (sp_cap < sp_bound) { + sp_cap <<= 1; + } + for (int32_t t = 0; t < active_sched_threads_; t++) { + sp_queues_[t].destroy(); // free a prior run's buffer before re-alloc + sp_queues_[t].init(sp_cap); + } + // Fold tasks completed inline during orchestration int32_t inline_completed = static_cast(rt->orchestrator.inline_completed_tasks); if (inline_completed > 0) { diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp index 96f17c18d..0791c3076 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_completion.cpp @@ -82,7 +82,7 @@ SlotTransition SchedulerContext::decide_slot_transition( // Complete one slot's task: subtask counting, mixed completion, deferred release, profiling. void SchedulerContext::complete_slot_task( PTO2TaskSlotState &slot_state, int32_t expected_reg_task_id, [[maybe_unused]] PTO2SubtaskSlot subslot, - [[maybe_unused]] int32_t thread_idx, int32_t core_id, Handshake *hank, int32_t &completed_this_turn + int32_t thread_idx, int32_t core_id, Handshake *hank, [[maybe_unused]] int32_t &completed_this_turn #if SIMPLER_DFX , uint64_t dispatch_ts, uint64_t finish_ts @@ -183,48 +183,15 @@ void SchedulerContext::complete_slot_task( ); } #endif + // 3S+1P: hand the finished task to the dedicated resolution (P) thread. + // P publishes completion_flags, drains the wake list, and advances the + // watermark — and owns completed_tasks_, so this scheduler thread neither + // resolves nor bumps completed_this_turn. (The Resolve swimlane bar is + // emitted by P, not here.) + sp_queues_[thread_idx].push(&slot_state); #if SIMPLER_DFX - // Time Resolve (walk the consumer list, decrement each consumer's - // fanin, push the newly-ready ones, ring doorbells for early-dispatch - // hits) so it renders as a child bar nested inside this iteration's - // Complete bar. The 1 µs floor below filters out the ~88% of tasks - // with 1-2 consumers (~500 ns Resolve) so only the long broadcast / - // reduction walks stand out on the lane. - uint64_t resolve_t0 = (l2_swimlane_level_ >= L2SwimlaneLevel::SCHED_PHASES) ? get_sys_cnt_aicpu() : 0; -#endif - // [[maybe_unused]] silences -Werror=unused-but-set-variable on the - // profiling-flags-smoke build path where SIMPLER_DFX is OFF and - // the Resolve emit below is excluded. - [[maybe_unused]] uint32_t consumers_resolved = 0; -#if SIMPLER_SCHED_PROFILING - // SCHED_PROFILING variant takes thread_idx for its per-thread atomic - // counter side-effects (g_sched_*_atomic_count[thread_idx], consumed - // by the otc_* log lines). It returns CompletionStats whose - // `fanout_edges` is the consumer-walk count. - consumers_resolved = sched_->on_task_complete(slot_state, thread_idx).fanout_edges; -#else - consumers_resolved = sched_->on_task_complete(slot_state); -#endif -#if SIMPLER_DFX - if (resolve_t0 != 0) { - uint64_t resolve_t1 = get_sys_cnt_aicpu(); - // Filter: drop Resolve bars under 1 µs so the lane shows only - // resolves that did meaningful work (high consumer counts or - // doorbells). 50 cycles @ 50 MHz = 1 µs (PLATFORM_PROF_SYS_CNT_FREQ - // is the device sys-cnt frequency). - constexpr uint64_t RESOLVE_EMIT_MIN_CYCLES = PLATFORM_PROF_SYS_CNT_FREQ / 1'000'000; // 1 µs - if (resolve_t1 - resolve_t0 >= RESOLVE_EMIT_MIN_CYCLES) { - l2_swimlane_aicpu_record_sched_phase( - thread_idx, L2SwimlaneSchedPhaseKind::Resolve, resolve_t0, resolve_t1, l2_swimlane.sched_loop_count, - consumers_resolved - ); - } - } l2_swimlane.phase_complete_count++; #endif - // Polling: on_task_complete published completion + drained the wake list - // inline; no deferred producer-release step. - completed_this_turn++; } #if SIMPLER_DFX diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h index 78f2eb1cc..96cc89992 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_context.h @@ -37,6 +37,57 @@ class Runtime; struct Handshake; struct PTO2Runtime; +// SPSC ring carrying completed-but-unresolved task slots from one scheduler (S) +// thread to the dedicated resolution (P) thread. Whole-graph-resident hbg never +// reclaims a slot, so the queued PTO2TaskSlotState* stays valid until P reads it. +// Single producer (the owning S thread) / single consumer (P): plain +// acquire/release on head/tail, no CAS. Capacity is a power of two sized to the +// in-flight bound (min of total tasks and the ring window), so a producer does +// not spin on a full queue in practice; the spin is a correctness backstop only +// (P drains independently, so it always makes progress). The std::atomic cursors +// make this type non-copyable / non-movable, so `buf` cannot be double-freed +// through an accidental copy. +struct CompletedTaskQueue { + PTO2TaskSlotState **buf{nullptr}; + uint64_t cap{0}; + uint64_t mask{0}; + alignas(64) std::atomic head{0}; // consumer (P) cursor + alignas(64) std::atomic tail{0}; // producer (S) cursor + + void init(uint64_t capacity_pow2) { + debug_assert(capacity_pow2 != 0 && (capacity_pow2 & (capacity_pow2 - 1)) == 0); + cap = capacity_pow2; + mask = capacity_pow2 - 1; + buf = new PTO2TaskSlotState *[capacity_pow2]; + head.store(0, std::memory_order_relaxed); + tail.store(0, std::memory_order_relaxed); + } + void destroy() { + delete[] buf; + buf = nullptr; + } + // Producer (S). Spins only if the ring is full — sized so this never fires. + void push(PTO2TaskSlotState *s) { + uint64_t t = tail.load(std::memory_order_relaxed); + while (t - head.load(std::memory_order_acquire) >= cap) { + SPIN_WAIT_HINT(); + } + buf[t & mask] = s; + tail.store(t + 1, std::memory_order_release); + } + // Consumer (P). Returns nullptr when empty. + PTO2TaskSlotState *pop() { + uint64_t h = head.load(std::memory_order_relaxed); + if (h == tail.load(std::memory_order_acquire)) { + return nullptr; + } + PTO2TaskSlotState *s = buf[h & mask]; + head.store(h + 1, std::memory_order_release); + return s; + } + uint64_t size() const { return tail.load(std::memory_order_acquire) - head.load(std::memory_order_acquire); } +}; + /** * SchedulerContext: owns all scheduler-side state and methods. * @@ -85,6 +136,16 @@ class SchedulerContext { // Main scheduler thread entry: poll completion + dispatch ready tasks. int32_t resolve_and_dispatch(Runtime *runtime, int32_t thread_idx); + // Dedicated resolution (P) thread entry (3S+1P). Owns no cores: drains the + // per-S CompletedTaskQueues and runs on_task_complete for each finished task + // (completion_flags publish + wake-list drain + watermark advance), making P + // the sole producer of the ready queues. Owns completed_tasks_ / termination. + int32_t run_resolution_thread(Runtime *runtime, int32_t thread_idx); + + // Index of the dedicated resolution (P) thread — the last AICPU thread. + // host_build_graph always reserves it (aicpu_thread_num >= 2 is required). + int32_t p_thread_idx() const { return p_thread_idx_; } + // Shutdown AICore registers for this thread's assigned cores. // Also runs PMU finalize (SIMPLER_DFX) before deinit when enabled. // Orchestrator threads (core_trackers_[thread_idx].core_num() == 0) are a no-op. @@ -157,6 +218,15 @@ class SchedulerContext { int32_t aicpu_thread_num_{0}; int32_t cores_total_num_{0}; + // --- 3S+1P dedicated resolution thread --- + // The AICPU threads split into (aicpu_thread_num_ - 1) core-owning schedulers + // (S) plus one core-less resolution thread (P) at index p_thread_idx_ = + // aicpu_thread_num_ - 1. host_build_graph requires aicpu_thread_num_ >= 2 (one + // S + one P). Each S thread hands its finished tasks to P through + // sp_queues_[its_thread_idx]. + int32_t p_thread_idx_{-1}; + CompletedTaskQueue sp_queues_[MAX_AICPU_THREADS]; + // Cluster-ordered worker_id lists, populated by post_handshake_init(). int32_t aic_worker_ids_[RUNTIME_MAX_WORKER]{}; int32_t aiv_worker_ids_[RUNTIME_MAX_WORKER]{}; diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp index 17fc4738e..66e6e53f6 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a2a3/runtime/host_build_graph/runtime/scheduler/scheduler_dispatch.cpp @@ -854,6 +854,183 @@ int32_t SchedulerContext::try_early_dispatch( return total_staged; } +// ============================================================================= +// Dedicated resolution (P) thread — 3S+1P +// ============================================================================= + +// P owns no AICore cores. It drains the per-S CompletedTaskQueues and runs +// on_task_complete for every finished task: publish completion_flags, drain the +// wake list (route/re-register waiters into the ready queues), advance the +// watermark. As the sole producer of the ready queues its enqueues never +// contend. P owns completed_tasks_ and the terminal completed_ flip, so the S +// threads keep dispatching until P has resolved the whole graph (watermark fully +// advanced) — the host's wait_for_consumers never observes a stranded prefix. +int32_t SchedulerContext::run_resolution_thread(Runtime *runtime, int32_t thread_idx) { + always_assert(sched_ != nullptr); + PTO2SharedMemoryHeader *header = sched_->sm_header; + if (!header) { + LOG_ERROR("PTO2 resolution: header is null"); + return -1; + } + LOG_INFO_V0("Thread %d: resolution (P) thread starting, serving %d schedulers", thread_idx, active_sched_threads_); + +#if SIMPLER_DFX + auto &l2_swimlane = sched_l2_swimlane_[thread_idx]; + l2_swimlane.reset(); + l2_swimlane.l2_swimlane_enabled = (l2_swimlane_level_ != L2SwimlaneLevel::DISABLED); +#endif + + uint64_t last_progress_ts = get_sys_cnt_aicpu(); + uint64_t scheduler_timeout_cycles = SCHEDULER_TIMEOUT_CYCLES; + const int32_t scheduler_timeout_ms_override = get_scheduler_timeout_ms(); + if (scheduler_timeout_ms_override > 0) { + scheduler_timeout_cycles = + static_cast(scheduler_timeout_ms_override) * PLATFORM_PROF_SYS_CNT_FREQ / 1000; + } + + while (true) { + if (completed_.load(std::memory_order_acquire)) break; + + // Propagate a fatal error latched by the orchestrator (host) or a + // scheduler thread; mirror resolve_and_dispatch's exit behavior. + if (header->orch_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE || + header->sched_error_code.load(std::memory_order_acquire) != PTO2_ERROR_NONE) { + if (!completed_.exchange(true, std::memory_order_acq_rel)) { + emergency_shutdown(runtime); + } + break; + } + + int32_t resolved_this_pass = 0; + for (int32_t s = 0; s < active_sched_threads_; s++) { + PTO2TaskSlotState *slot; + while ((slot = sp_queues_[s].pop()) != nullptr) { +#if SIMPLER_SCHED_PROFILING + (void)sched_->on_task_complete(*slot, thread_idx); +#else + (void)sched_->on_task_complete(*slot); +#endif + resolved_this_pass++; + } + } + + // Async deferred completions, moved off the scheduler threads. Every + // condition that fires resolves via on_task_complete inside + // poll_and_complete, so async ready tasks also enter the ready queues + // through P alone. + if (rt_ != nullptr && rt_->aicore_mailbox != nullptr && + (sched_->async_wait_list.count > 0 || rt_->aicore_mailbox->has_pending())) { + AsyncPollResult poll_result = sched_->async_wait_list.poll_and_complete( + rt_->aicore_mailbox, sched_ +#if SIMPLER_SCHED_PROFILING + , + thread_idx +#endif + ); + if (poll_result.error_code != PTO2_ERROR_NONE) { + int32_t expected = PTO2_ERROR_NONE; + header->sched_error_code.compare_exchange_strong( + expected, poll_result.error_code, std::memory_order_acq_rel, std::memory_order_acquire + ); + if (!completed_.exchange(true, std::memory_order_acq_rel)) { + emergency_shutdown(runtime); + } + break; + } + resolved_this_pass += poll_result.completed; + } + + // Dependency-only tasks (empty active_mask, or a predicate that failed) + // route to dummy_ready_queue during resolution; P produces and drains it, + // so the queue is single-threaded end to end. Loop until empty — a dummy's + // resolution can make further dummies ready in the same pass. + { + constexpr int DUMMY_DRAIN_BATCH = 8; + PTO2TaskSlotState *dummy_batch[DUMMY_DRAIN_BATCH]; + int dummy_got; + while ((dummy_got = sched_->dummy_ready_queue.pop_batch(dummy_batch, DUMMY_DRAIN_BATCH)) > 0) { + for (int di = 0; di < dummy_got; di++) { +#if SIMPLER_SCHED_PROFILING + (void)sched_->on_task_complete(*dummy_batch[di], thread_idx); +#else + (void)sched_->on_task_complete(*dummy_batch[di]); +#endif + resolved_this_pass++; + } + } + } + + if (resolved_this_pass > 0) { + int32_t new_total = + completed_tasks_.fetch_add(resolved_this_pass, std::memory_order_relaxed) + resolved_this_pass; +#if SIMPLER_SCHED_PROFILING + // P owns the completion accounting, so it owns the profiling mirror too + // (the S threads' completed_this_turn no longer feeds it in P mode). + sched_->tasks_completed.fetch_add(resolved_this_pass, std::memory_order_relaxed); +#endif + last_progress_ts = get_sys_cnt_aicpu(); + if (total_tasks_ > 0 && new_total >= total_tasks_) { + completed_.store(true, std::memory_order_release); + LOG_INFO_V0("Thread %d: P resolved all tasks %d/%d", thread_idx, new_total, total_tasks_); + break; + } + continue; // fast re-drain while work keeps arriving + } + + // Idle: nothing to resolve this pass. A task legitimately in flight — some + // thread still owns a RUNNING core — means P is merely waiting for that + // task to finish, not stalled: refresh the budget and keep spinning + // (mirrors resolve_and_dispatch's sibling-owns-running guard, so a task + // that runs longer than the timeout does not false-latch here). Only latch + // a hang when work is outstanding AND no thread anywhere owns a running + // task — a genuine forward-progress stall / pre-dispatch deadlock. + uint64_t now = get_sys_cnt_aicpu(); + if (now - last_progress_ts > scheduler_timeout_cycles) { + bool outstanding = total_tasks_ > 0 && completed_tasks_.load(std::memory_order_relaxed) < total_tasks_; + if (outstanding && no_thread_owns_running_task()) { + LOG_ERROR( + "Thread %d: P resolution stall (%d/%d resolved)", thread_idx, + completed_tasks_.load(std::memory_order_relaxed), total_tasks_ + ); + int32_t expected = PTO2_ERROR_NONE; + header->sched_error_code.compare_exchange_strong( + expected, PTO2_ERROR_SCHEDULER_TIMEOUT, std::memory_order_acq_rel, std::memory_order_acquire + ); + if (!completed_.exchange(true, std::memory_order_acq_rel)) { + emergency_shutdown(runtime); + } + break; + } + last_progress_ts = now; // a task is still running (or none outstanding): not a stall + } + SPIN_WAIT_HINT(); + } + +#if SIMPLER_DFX + // P owns no cores, so the AICore-keyed flushes below iterate an empty core + // list; the sched-phase-buffer flush is the one that matters — it drains any + // per-thread records P wrote (e.g. under SCHED_PROFILING) so they are not lost. + if (l2_swimlane.l2_swimlane_enabled) { + l2_swimlane_aicpu_flush( + thread_idx, core_trackers_[thread_idx].core_ids(), core_trackers_[thread_idx].core_num() + ); + if (l2_swimlane_level_ >= L2SwimlaneLevel::SCHED_PHASES) { + l2_swimlane_aicpu_flush_sched_phase_buffer(thread_idx); + } + } + if (is_dump_args_enabled()) { + dump_args_flush(thread_idx); + } + if (is_pmu_enabled()) { + pmu_aicpu_flush_buffers( + thread_idx, core_trackers_[thread_idx].core_ids(), core_trackers_[thread_idx].core_num() + ); + } +#endif + + return completed_tasks_.load(std::memory_order_relaxed); +} + // ============================================================================= // Main scheduler dispatch loop // ============================================================================= @@ -1031,33 +1208,11 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ } } - if (rt_ != nullptr && rt_->aicore_mailbox != nullptr && - (sched_->async_wait_list.count > 0 || rt_->aicore_mailbox->has_pending())) { - AsyncPollResult poll_result = sched_->async_wait_list.poll_and_complete( - rt_->aicore_mailbox, sched_ -#if SIMPLER_SCHED_PROFILING - , - thread_idx -#endif - ); - if (poll_result.error_code != PTO2_ERROR_NONE) { - int32_t expected = PTO2_ERROR_NONE; - header->sched_error_code.compare_exchange_strong( - expected, poll_result.error_code, std::memory_order_acq_rel, std::memory_order_acquire - ); - completed_.store(true, std::memory_order_release); - break; - } - if (poll_result.completed > 0) { -#if SIMPLER_SCHED_PROFILING - sched_->tasks_completed.fetch_add(poll_result.completed, std::memory_order_relaxed); -#endif - int32_t prev = completed_tasks_.fetch_add(poll_result.completed, std::memory_order_relaxed); - int32_t new_total = prev + poll_result.completed; - last_progress_count = new_total; - made_progress = true; - } - } + // Async deferred-completion polling and dependency-only (dummy / + // predicate-failed) retirement both run on P, which owns every + // completion→ready transition — the scheduler threads' loop stays purely + // core-local (poll own COND, dispatch own cores) and never touches the + // shared mailbox or dummy queue. #if SIMPLER_DFX if (!try_completed) { @@ -1116,97 +1271,9 @@ int32_t SchedulerContext::resolve_and_dispatch(Runtime *runtime, int32_t thread_ continue; } - // Phase 3: Drain dummy ready queue (S0/S1/S2). - // - // Dependency-only tasks bypass AICore dispatch: they go through the - // scheduler so fanin/fanout edges stay consistent, but completion is - // signalled inline here. The ready queue is MPMC, and the fanout path - // uses per-slot locks/atomics, so multiple scheduler threads can share - // the dependency-only resolve work. - if (thread_idx < 3) { - constexpr int DUMMY_DRAIN_BATCH = 8; - PTO2TaskSlotState *dummy_batch[DUMMY_DRAIN_BATCH]; - int dummy_got = sched_->dummy_ready_queue.pop_batch(dummy_batch, DUMMY_DRAIN_BATCH); -#if SIMPLER_DFX - // Dummy outer phase: covers handling of all dummies popped this - // iter. Per-dummy DummyTask markers are emitted to a SEPARATE lane - // (Worker View AICPU_N) by the converter, so they do not nest - // under this bar. Resolve emits below DO land on the sched lane - // and nest under this Dummy outer by time containment. - uint64_t dummy_outer_t0 = - (dummy_got > 0 && l2_swimlane_level_ >= L2SwimlaneLevel::SCHED_PHASES) ? get_sys_cnt_aicpu() : 0; -#endif - for (int di = 0; di < dummy_got; di++) { - PTO2TaskSlotState &dummy_slot = *dummy_batch[di]; - - // ----- Resolve work: walk this dummy's consumer list. ------ - // Same 1 µs filter as the main-path Resolve emit suppresses - // dummies whose consumer release runs sub-microsecond. -#if SIMPLER_DFX - uint64_t dummy_resolve_t0 = - (l2_swimlane_level_ >= L2SwimlaneLevel::SCHED_PHASES) ? get_sys_cnt_aicpu() : 0; -#endif - // [[maybe_unused]] silences -Werror=unused-but-set-variable on - // the profiling-flags-smoke build path where SIMPLER_DFX is - // OFF and the Resolve emit below is excluded. - [[maybe_unused]] uint32_t dummy_consumers = 0; -#if SIMPLER_SCHED_PROFILING - dummy_consumers = sched_->on_task_complete(dummy_slot, thread_idx).fanout_edges; -#else - dummy_consumers = sched_->on_task_complete(dummy_slot); -#endif -#if SIMPLER_DFX - if (dummy_resolve_t0 != 0) { - uint64_t dummy_resolve_t1 = get_sys_cnt_aicpu(); - constexpr uint64_t RESOLVE_EMIT_MIN_CYCLES = PLATFORM_PROF_SYS_CNT_FREQ / 1'000'000; // 1 µs - if (dummy_resolve_t1 - dummy_resolve_t0 >= RESOLVE_EMIT_MIN_CYCLES) { - l2_swimlane_aicpu_record_sched_phase( - thread_idx, L2SwimlaneSchedPhaseKind::Resolve, dummy_resolve_t0, dummy_resolve_t1, - sched_l2_swimlane_[thread_idx].sched_loop_count, dummy_consumers - ); - } - l2_swimlane_aicpu_record_dummy_task( - thread_idx, dummy_resolve_t0, sched_l2_swimlane_[thread_idx].sched_loop_count, - dummy_slot.task->task_id.raw - ); - } -#endif - // Polling: on_task_complete already published this slot's - // completion + drained its wake list inline. There is no deferred - // producer-release phase — consumer retirement is observed via the - // per-ring completed_watermark, not by bumping producer refcounts. - int32_t prev = completed_tasks_.fetch_add(1, std::memory_order_relaxed); - last_progress_count = prev + 1; - cur_thread_completed++; - } - if (dummy_got > 0) { - made_progress = true; - } -#if SIMPLER_DFX - // Emit Dummy outer over the whole dummy_drain pass. Span starts at - // dummy_outer_t0 (captured after pop_batch) and ends at "now". - // tasks_processed = dummy_got. Advancing _t0_phase here makes the - // following Dispatch / EarlyDispatch / second-Complete bars start - // at this end. - if (dummy_outer_t0 != 0) { - uint64_t dummy_outer_t1 = get_sys_cnt_aicpu(); - int16_t phase_end_shared[L2SWIMLANE_NUM_QUEUE_SHAPES]; - capture_phase_end_fresh(phase_end_shared); - l2_swimlane_aicpu_record_sched_phase( - thread_idx, L2SwimlaneSchedPhaseKind::Dummy, dummy_outer_t0, dummy_outer_t1, - l2_swimlane.sched_loop_count, static_cast(dummy_got), /*pop_hit=*/0, - /*pop_miss=*/0, phase_start_shared, phase_end_shared - ); - for (int s = 0; s < L2SWIMLANE_NUM_QUEUE_SHAPES; s++) - phase_start_shared[s] = phase_end_shared[s]; - _t0_phase = dummy_outer_t1; - // We do NOT re-sync _t0/_t1 — the dummy span will be absorbed - // into the next CYCLE_COUNT_LAP accumulator. The phase-model - // anchor (_t0_phase) is the authoritative source for bar spans - // on the swimlane; the cycle accumulators are coarse aggregates. - } -#endif - } + // Phase 3 (dependency-only dummy / predicate-failed retirement) runs on + // the resolution thread P, not here — see run_resolution_thread. The + // scheduler loop goes straight from completion detection to dispatch. // Phase 4: MIX-strict-priority dispatch with phase-split and // cross-thread idle gating. See dispatch_ready_tasks for the policy.