diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 25b69e934..dd9c14155 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -634,7 +634,7 @@ pending slot, promoted on completion). a5 implements this order inline in | NORMAL (ready) | `ready_queues[MIX\|AIC\|AIV]` | *(same `ready_queues[]` — a5 has no `ready_sync_queues[]`)* | | EARLY (speculative) | `early_dispatch_queues[MIX\|AIC\|AIV]` | `early_sync_start_queue` (single) | -A task routes to the early sync lane iff `active_mask.requires_sync_start()`. Early +A task routes to the early sync lane iff `task_attrs.requires_sync_start()`. Early dispatch runs only once normal `ready_queues[]` are empty **and** the local `CoreTracker` has a spare slot (`has_any_free_slot`, a2a3 #1288). diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp index 625289819..fa352aede 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp @@ -362,7 +362,7 @@ static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) { static bool all_claimed_fanin_allow_early_resolve(const PTO2FaninBuilder &fanin_builder) { if (fanin_builder.count == 0) return true; return fanin_builder.for_each([](PTO2TaskSlotState *producer) -> bool { - return producer != nullptr && producer->allow_early_resolve; + return producer != nullptr && producer->task_attrs.allow_early_resolve(); }); } @@ -390,7 +390,7 @@ void PTO2OrchestratorState::wire_fanin_task(PTO2TaskSlotState &slot_state, int32 for_each_fanin_slot_state(*payload, [&](PTO2TaskSlotState *producer) { producer->lock_fanout(); int32_t pstate = producer->task_state.load(std::memory_order_acquire); - if (!early_disqualified && !producer->allow_early_resolve) { + if (!early_disqualified && !producer->task_attrs.allow_early_resolve()) { early_disqualified = true; } if (pstate >= PTO2_TASK_COMPLETED) { @@ -517,7 +517,7 @@ static void prefetch_payload(PTO2TaskPayload *payload, int32_t tensor_count, int static bool prepare_task( PTO2OrchestratorState *orch, const L0TaskArgs &args, int32_t total_output_size, ActiveMask active_mask, - PTO2PreparedTask *out + TaskAttrs task_attrs, PTO2PreparedTask *out ) { uint8_t ring_id = orch->current_ring_id(); auto &allocator = orch->rings[ring_id].task_allocator; @@ -576,6 +576,7 @@ static bool prepare_task( static_cast(block_num * __builtin_popcount(active_mask.core_mask())); out->slot_state->logical_block_num = block_num; out->slot_state->active_mask = active_mask; + out->slot_state->task_attrs = task_attrs; // fanin_count is set during Orch-side wiring scope_tasks_push(orch, out->slot_state); @@ -798,14 +799,14 @@ static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t neede // computation (explicit_deps + auto), output registration, slot init, and // Orch-side wiring/ready publication. static TaskOutputTensors submit_task_common( - PTO2OrchestratorState *orch, const L0TaskArgs &args, ActiveMask active_mask, int32_t aic_kernel_id, - int32_t aiv0_kernel_id, int32_t aiv1_kernel_id + PTO2OrchestratorState *orch, const L0TaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs, + int32_t aic_kernel_id, int32_t aiv0_kernel_id, int32_t aiv1_kernel_id ) { CYCLE_COUNT_START(); TaskOutputTensors result; PTO2OutputLayout layout = calculate_output_layout(args); PTO2PreparedTask prepared; - if (!prepare_task(orch, args, layout.total_output_size, active_mask, &prepared)) { + if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) { return result; } uint8_t ring_id = prepared.task_id.ring(); @@ -937,7 +938,6 @@ static TaskOutputTensors submit_task_common( task.kernel_id[static_cast(PTO2SubtaskSlot::AIC)] = aic_kernel_id; task.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)] = aiv0_kernel_id; task.kernel_id[static_cast(PTO2SubtaskSlot::AIV1)] = aiv1_kernel_id; - task.task_timing_slot = args.task_timing_slot(); task.packed_buffer_base = prepared.alloc_result.packed_base; task.packed_buffer_end = prepared.alloc_result.packed_end; @@ -966,7 +966,6 @@ static TaskOutputTensors submit_task_common( } payload.init(args, result, prepared.alloc_result, layout); - cur_slot_state.set_allow_early_resolve(args.allow_early_resolve()); // Dispatch predicate: resolve the (tensor, indices) to an absolute GM address // now so the scheduler can read it at the dispatch point with a single load, @@ -1086,7 +1085,11 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k active_mask = normalized.to_active_mask(); } - // Encode require_sync_start into active_mask bit 3 (only meaningful for tasks with block_num > 1) + TaskAttrs task_attrs; + task_attrs.set_early_resolve(args.allow_early_resolve()); + task_attrs.set_timing_slot(args.task_timing_slot()); + + // sync_start is only meaningful for tasks with block_num > 1. if (block_num > 1 && args.launch_spec.require_sync_start()) { // Deadlock check: block_num >= total available slots of the required type. // For MIX/AIC: limit is total_cluster_count (one AIC per cluster). @@ -1100,15 +1103,16 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k ); return TaskOutputTensors{}; } - active_mask.set_sync_start(); + task_attrs.set_sync_start(); } if (args.predicate().op != PredicateOp::NONE) { - active_mask.set_has_predicate(); + task_attrs.set_predicate(); } return submit_task_common( - orch, args, active_mask, normalized.aic_kernel_id, normalized.aiv0_kernel_id, normalized.aiv1_kernel_id + orch, args, active_mask, task_attrs, normalized.aic_kernel_id, normalized.aiv0_kernel_id, + normalized.aiv1_kernel_id ); } @@ -1136,7 +1140,15 @@ TaskOutputTensors PTO2OrchestratorState::submit_dummy_task(const L0TaskArgs &arg } always_assert(orch->scheduler != nullptr); - return submit_task_common(orch, args, ActiveMask{}, INVALID_KERNEL_ID, INVALID_KERNEL_ID, INVALID_KERNEL_ID); + // Dummy tasks never dispatch to an AICore, so sync_start / has_predicate do + // not apply; only the early-dispatch hint and timing tag carry over. + TaskAttrs task_attrs; + task_attrs.set_early_resolve(args.allow_early_resolve()); + task_attrs.set_timing_slot(args.task_timing_slot()); + + return submit_task_common( + orch, args, ActiveMask{}, task_attrs, INVALID_KERNEL_ID, INVALID_KERNEL_ID, INVALID_KERNEL_ID + ); } TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const L0TaskArgs &args) { @@ -1176,7 +1188,9 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const L0TaskArgs &args) { PTO2OutputLayout layout = calculate_output_layout(args); PTO2PreparedTask prepared; - if (!prepare_task(orch, args, layout.total_output_size, ActiveMask{}, &prepared)) { + // Kernel-less alloc task: no active subtasks, no dispatch-time attributes. The + // early-dispatch hint is force-set below (see the flag-the-creator note). + if (!prepare_task(orch, args, layout.total_output_size, ActiveMask{}, TaskAttrs{}, &prepared)) { return TaskOutputTensors{}; } @@ -1196,9 +1210,6 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const L0TaskArgs &args) { task.kernel_id[static_cast(PTO2SubtaskSlot::AIC)] = INVALID_KERNEL_ID; task.kernel_id[static_cast(PTO2SubtaskSlot::AIV0)] = INVALID_KERNEL_ID; task.kernel_id[static_cast(PTO2SubtaskSlot::AIV1)] = INVALID_KERNEL_ID; - // alloc_tensors builds a kernel-less descriptor that never dispatches; keep - // the slot untagged so a recycled ring slot cannot leak a stale tag. - task.task_timing_slot = TASK_TIMING_SLOT_NONE; task.packed_buffer_base = prepared.alloc_result.packed_base; task.packed_buffer_end = prepared.alloc_result.packed_end; @@ -1227,7 +1238,7 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const L0TaskArgs &args) { // creation — it should always be transparent, never a barrier. Unlike a // codegen task there is no Arg-driven hint to honor here, so mark it // unconditionally. (a2a3 #1285/#1292) - prepared.slot_state->allow_early_resolve = true; + prepared.slot_state->task_attrs.set_early_resolve(true); prepared.slot_state->mark_completed(); } orch->inline_completed_tasks++; diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h index 25d433c84..5c7de9199 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_runtime2_types.h @@ -197,26 +197,14 @@ struct PTO2TaskDescriptor { // Per-slot kernel IDs (INVALID_KERNEL_ID = inactive) int32_t kernel_id[PTO2_SUBTASK_SLOT_COUNT]; - // Selective task-timing slot (TASK_TIMING_SLOT_NONE = untagged; 0..15 valid). - // Occupies the 4-byte alignment pad that already followed kernel_id[3], so - // the descriptor does not grow; the scheduler folds this task's dispatch/ - // finish cycles into the tagged slot (see common/device_phase.h). - int32_t task_timing_slot; - // Packed output buffer (all outputs packed into single contiguous buffer) void *packed_buffer_base; // Start of packed buffer in GM Heap void *packed_buffer_end; // End of packed buffer (for heap reclamation) }; -// task_timing_slot must occupy the former padding after kernel_id[3] without -// growing the descriptor or shifting packed_buffer_base — the scheduler and -// shared-memory ABI depend on the existing size and pointer offset. -static_assert(sizeof(PTO2TaskDescriptor) == 40, "PTO2TaskDescriptor must not grow: slot uses the existing pad"); -static_assert( - offsetof(PTO2TaskDescriptor, task_timing_slot) == - offsetof(PTO2TaskDescriptor, kernel_id) + sizeof(int32_t) * PTO2_SUBTASK_SLOT_COUNT, - "task_timing_slot must sit immediately after kernel_id in the former pad" -); +// A 4-byte alignment pad follows kernel_id[3]; the scheduler and shared-memory +// ABI depend on the descriptor size and packed_buffer_base offset staying fixed. +static_assert(sizeof(PTO2TaskDescriptor) == 40, "PTO2TaskDescriptor size is part of the shared-memory ABI"); static_assert(offsetof(PTO2TaskDescriptor, packed_buffer_base) == 24, "packed_buffer_base offset must be unchanged"); // ============================================================================= @@ -445,9 +433,14 @@ struct alignas(64) PTO2TaskSlotState { // --- Set per-submit (depend on task inputs) --- ActiveMask active_mask; // Bitmask of active subtask slots (set once) uint8_t ring_id; // Ring layer (immutable after init) - // These one-byte flags live in the padding before dep_pool_mark to keep - // PTO2TaskSlotState at 64 bytes. - bool allow_early_resolve{false}; + // Single per-task attributes byte (early-dispatch hint, sync_start, + // has_predicate, selective timing tag). Lives on slot_state (not payload) so + // fanin walks and the completion path read them off the already-hot producer + // slot_state cache line. Packed into the padding before dep_pool_mark to keep + // PTO2TaskSlotState at 64 bytes. Plain-write (set once at submit, before the + // slot is scheduler-visible), so it MUST NOT share a byte with the atomically + // mutated lifecycle_flags. + TaskAttrs task_attrs{}; std::atomic lifecycle_flags{PTO2_LIFECYCLE_FLAGS_NONE}; int32_t dep_pool_mark{0}; // Dep pool top after Orch-side wiring @@ -521,8 +514,6 @@ struct alignas(64) PTO2TaskSlotState { return (lifecycle_flags.load(std::memory_order_acquire) & PTO2_SUBTASK_DEFERRED) != 0; } - void set_allow_early_resolve(bool v) { allow_early_resolve = v; } - /** * Reset dynamic scheduling fields for slot reuse. * Called by advance_ring_pointers() after a slot transitions to CONSUMED @@ -543,7 +534,8 @@ struct alignas(64) PTO2TaskSlotState { completed_subtasks.store(0, std::memory_order_relaxed); next_block_idx.store(0, std::memory_order_relaxed); lifecycle_flags.store(PTO2_LIFECYCLE_FLAGS_NONE, std::memory_order_relaxed); - allow_early_resolve = false; + // Note: active_mask and task_attrs are per-submit-constant fields + // rewritten in prepare_task on every reuse, so they are not reset here. } // === Per-task fanout spinlock === diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_submit_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_submit_types.h index 360ae47b9..d594af230 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_submit_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_submit_types.h @@ -39,11 +39,9 @@ enum class PTO2SubtaskSlot : uint8_t { /** * Subtask mask bits (for ActiveMask) */ -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIC = (1u << 0); // 0x1 -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV0 = (1u << 1); // 0x2 -inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV1 = (1u << 2); // 0x4 -inline constexpr uint8_t PTO2_SUBTASK_FLAG_SYNC_START = (1u << 3); // 0x8: all blocks must launch atomically -inline constexpr uint8_t PTO2_SUBTASK_FLAG_HAS_PREDICATE = (1u << 4); // 0x10: task carries a dispatch predicate +inline constexpr uint8_t PTO2_SUBTASK_MASK_AIC = (1u << 0); // 0x1 +inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV0 = (1u << 1); // 0x2 +inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV1 = (1u << 2); // 0x4 // Dispatch-predicate comparison operator. The scheduler evaluates the predicate // at the dispatch point — the task is ready (fanin satisfied), so the predicate @@ -120,7 +118,11 @@ enum class PTO2ResourceShape : uint8_t { inline constexpr int32_t PTO2_NUM_RESOURCE_SHAPES = 3; /** - * Bitmask of active subtask slots + flags, sizeof == 1. + * Bitmask of active subtask slots (AIC/AIV0/AIV1), sizeof == 1. + * + * Pure subtask-slot mask: only the low 3 bits (core_mask) are meaningful, so it + * can be OR/==-combined when building MIX clusters without dragging unrelated + * flag bits along. Per-task scheduling flags live on TaskAttrs instead. */ class ActiveMask { public: @@ -134,11 +136,6 @@ class ActiveMask { uint8_t core_mask() const { return raw_ & 0x07u; } - bool requires_sync_start() const { return (raw_ & PTO2_SUBTASK_FLAG_SYNC_START) != 0; } - - // True when the task carries a dispatch predicate. - bool has_predicate() const { return (raw_ & PTO2_SUBTASK_FLAG_HAS_PREDICATE) != 0; } - PTO2ResourceShape to_shape() const { uint8_t cmask = core_mask(); if (cmask == 0) return PTO2ResourceShape::DUMMY; @@ -148,10 +145,6 @@ class ActiveMask { return PTO2ResourceShape::AIV; } - void set_sync_start() { raw_ |= PTO2_SUBTASK_FLAG_SYNC_START; } - - void set_has_predicate() { raw_ |= PTO2_SUBTASK_FLAG_HAS_PREDICATE; } - bool operator==(ActiveMask other) const { return raw_ == other.raw_; } bool operator!=(ActiveMask other) const { return raw_ != other.raw_; } @@ -173,6 +166,67 @@ class ActiveMask { static_assert(sizeof(ActiveMask) == 1, "ActiveMask must be exactly 1 byte"); +/** + * Per-task scheduling attributes, packed into one byte on PTO2TaskSlotState. + * + * Single home for the independent per-task flags: an early-dispatch hint, the + * two dispatch-time predicates (sync_start / has_predicate), and the selective + * timing tag. Consolidating them here keeps active_mask a pure subtask-slot mask + * and lands the timing tag on the scheduler's hot slot_state cache line. + * + * bit 0 allow_early_resolve + * bit 1 sync_start + * bit 2 has_predicate + * bit 3 is_timed (0 => untagged; timing_tag ignored) + * bits 4-7 timing_tag (0..15) + */ +class TaskAttrs { +public: + constexpr TaskAttrs() = default; + + bool allow_early_resolve() const { return (raw_ & BIT_EARLY_RESOLVE) != 0; } + void set_early_resolve(bool v) { + if (v) { + raw_ |= BIT_EARLY_RESOLVE; + } else { + raw_ &= static_cast(~BIT_EARLY_RESOLVE); + } + } + + bool requires_sync_start() const { return (raw_ & BIT_SYNC_START) != 0; } + void set_sync_start() { raw_ |= BIT_SYNC_START; } + + bool has_predicate() const { return (raw_ & BIT_HAS_PREDICATE) != 0; } + void set_predicate() { raw_ |= BIT_HAS_PREDICATE; } + + bool is_timed() const { return (raw_ & BIT_IS_TIMED) != 0; } + + // 0..15 timing tag when tagged, else -1 (matches TASK_TIMING_SLOT_NONE; the + // equality is guarded by a static_assert in pto_types.h). + int32_t timing_slot() const { return is_timed() ? static_cast(raw_ >> TIMING_TAG_SHIFT) : -1; } + + // slot < 0 => untagged; 0..15 => tagged. Arg::set_task_timing_slot already + // range-checks 0..NUM_TASK_TIMING_SLOTS-1, so only the low 4 bits are stored. + void set_timing_slot(int32_t slot) { + raw_ &= static_cast(~(BIT_IS_TIMED | TIMING_TAG_MASK)); + if (slot >= 0) { + raw_ |= static_cast(BIT_IS_TIMED | ((slot & 0x0F) << TIMING_TAG_SHIFT)); + } + } + +private: + static constexpr uint8_t BIT_EARLY_RESOLVE = 1u << 0; + static constexpr uint8_t BIT_SYNC_START = 1u << 1; + static constexpr uint8_t BIT_HAS_PREDICATE = 1u << 2; + static constexpr uint8_t BIT_IS_TIMED = 1u << 3; + static constexpr uint8_t TIMING_TAG_SHIFT = 4; + static constexpr uint8_t TIMING_TAG_MASK = 0xF0u; + + uint8_t raw_{0}; +}; + +static_assert(sizeof(TaskAttrs) == 1, "TaskAttrs must be exactly 1 byte"); + /** * Mixed-task submit contract. * diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_types.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_types.h index 27829ffb6..8835c616d 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_types.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_types.h @@ -45,6 +45,11 @@ #include "tensor.h" #include "tensor_create_info.h" // runtime-only TensorCreateInfo + materialization helpers +// TaskAttrs packs the timing tag into a 4-bit field and reports "untagged" as +// -1, so the tag domain must fit 0..15 and the untagged sentinel must be -1. +static_assert(NUM_TASK_TIMING_SLOTS <= 16, "timing tag must fit TaskAttrs' 4-bit field"); +static_assert(TASK_TIMING_SLOT_NONE == -1, "TaskAttrs::timing_slot() reports untagged as -1"); + typedef enum { ASYNC_ENGINE_SDMA = 0, ASYNC_ENGINE_ROCE = 1, diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h index 61e59215d..d5b74aa46 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/pto_scheduler.h @@ -525,7 +525,7 @@ struct PTO2SchedulerState { void push_ready_routed(PTO2TaskSlotState *slot_state) { PTO2ResourceShape shape = slot_state->active_mask.to_shape(); if (shape == PTO2ResourceShape::DUMMY || - (slot_state->active_mask.has_predicate() && !slot_state->payload->predicate.pass())) { + (slot_state->task_attrs.has_predicate() && !slot_state->payload->predicate.pass())) { dummy_ready_queue.push(slot_state); } else { ready_queues[static_cast(shape)].push(slot_state); @@ -696,7 +696,7 @@ struct PTO2SchedulerState { } inline void record_published_blocks(PTO2TaskSlotState &slot_state, int32_t count) { - if (count <= 0 || !slot_state.allow_early_resolve) return; + if (count <= 0 || !slot_state.task_attrs.allow_early_resolve()) return; slot_state.payload->published_block_count.fetch_add(static_cast(count), std::memory_order_seq_cst); } @@ -784,7 +784,7 @@ struct PTO2SchedulerState { } inline void try_enqueue_early_dispatch_candidate(PTO2TaskSlotState &consumer) { - if (consumer.active_mask.has_predicate()) return; + if (consumer.task_attrs.has_predicate()) return; PTO2ResourceShape shape = consumer.active_mask.to_shape(); if (shape != PTO2ResourceShape::AIC && shape != PTO2ResourceShape::AIV && shape != PTO2ResourceShape::MIX) { return; @@ -798,7 +798,7 @@ struct PTO2SchedulerState { } uint64_t task_id = static_cast(consumer.task->task_id.raw); - if (consumer.active_mask.requires_sync_start()) { + if (consumer.task_attrs.requires_sync_start()) { early_sync_start_queue.push_tagged(&consumer, task_id); } else { early_dispatch_queues[static_cast(shape)].push_tagged(&consumer, task_id); @@ -809,12 +809,12 @@ struct PTO2SchedulerState { // early-dispatch off its DIRECT producers' marks, never an inherited chain // (a2a3 #1285 — a5 never had auto-chain / spec_chain_*). void propagate_dispatch_fanin(PTO2TaskSlotState &p) { - if (!p.allow_early_resolve) return; + if (!p.task_attrs.allow_early_resolve()) return; if (p.payload->published_block_count.load(std::memory_order_seq_cst) < p.logical_block_num) return; if (p.payload->early_dispatch_state.load(std::memory_order_acquire) == PTO2_EARLY_DISPATCH_STAGING) return; uint8_t launch_state = p.payload->early_dispatch_launch_state.load(std::memory_order_acquire); if (launch_state == PTO2_EARLY_DISPATCH_LAUNCH_RINGING) return; - if (p.active_mask.requires_sync_start()) { + if (p.task_attrs.requires_sync_start()) { bool was_pre_staged = false; for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; w++) { was_pre_staged |= p.payload->staged_core_mask[w].load(std::memory_order_acquire) != 0; @@ -831,7 +831,7 @@ struct PTO2SchedulerState { p.unlock_fanout(); for (; edge != nullptr; edge = edge->next) { PTO2TaskSlotState *c = edge->slot_state; - if (c->active_mask.has_predicate()) continue; + if (c->task_attrs.has_predicate()) continue; int32_t nf = c->payload->dispatch_fanin.fetch_add(1, std::memory_order_acq_rel) + 1; if (nf != c->payload->fanin_actual_count) continue; try_enqueue_early_dispatch_candidate(*c); @@ -850,7 +850,7 @@ struct PTO2SchedulerState { }; inline bool try_early_dispatch_release(PTO2TaskSlotState &slot_state, EarlyDispatchReleaseSink *sink = nullptr) { - if (slot_state.active_mask.has_predicate()) return false; + if (slot_state.task_attrs.has_predicate()) return false; uint8_t expect = PTO2_EARLY_DISPATCH_NONE; if (slot_state.payload->early_dispatch_state.compare_exchange_strong( expect, PTO2_EARLY_DISPATCH_DISPATCHED, std::memory_order_seq_cst, std::memory_order_seq_cst @@ -858,7 +858,7 @@ struct PTO2SchedulerState { return false; } if (expect != PTO2_EARLY_DISPATCH_STAGING) return true; - bool sync_start = slot_state.active_mask.requires_sync_start(); + bool sync_start = slot_state.task_attrs.requires_sync_start(); if (!sync_start && !try_claim_early_dispatch_launch(*slot_state.payload)) return true; expect = PTO2_EARLY_DISPATCH_STAGING; slot_state.payload->early_dispatch_state.compare_exchange_strong( @@ -901,7 +901,7 @@ struct PTO2SchedulerState { bool route_ready_once(PTO2TaskSlotState &slot_state, EarlyDispatchReleaseSink *sink = nullptr) { if (!try_claim_ready_once(slot_state)) return false; bool early_handled = try_early_dispatch_release(slot_state, sink); - if (slot_state.active_mask.requires_sync_start()) { + if (slot_state.task_attrs.requires_sync_start()) { bool drain_owned = publish_ready_to_early_sync_drain(*slot_state.payload); if (early_handled || drain_owned) return true; } else if (early_handled) { @@ -930,7 +930,7 @@ struct PTO2SchedulerState { atomic_count += 1; } bool early_handled = try_early_dispatch_release(slot_state, sink); - if (slot_state.active_mask.requires_sync_start()) { + if (slot_state.task_attrs.requires_sync_start()) { bool drain_owned = publish_ready_to_early_sync_drain(*slot_state.payload); if (early_handled || drain_owned) return true; } else if (early_handled) { @@ -938,7 +938,7 @@ struct PTO2SchedulerState { } PTO2ResourceShape shape = slot_state.active_mask.to_shape(); if (shape == PTO2ResourceShape::DUMMY || - (slot_state.active_mask.has_predicate() && !slot_state.payload->predicate.pass())) { + (slot_state.task_attrs.has_predicate() && !slot_state.payload->predicate.pass())) { dummy_ready_queue.push(&slot_state, atomic_count, push_wait); } else { ready_queues[static_cast(shape)].push(&slot_state, atomic_count, push_wait); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp index 7a65524aa..f1d74b9cd 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_completion.cpp @@ -299,7 +299,7 @@ void SchedulerContext::check_running_cores_for_completion( (core.pending_slot_state != nullptr && core.pending_slot_state->payload != nullptr && (pending_ss == PTO2_EARLY_DISPATCH_STAGING || (pending_ss == PTO2_EARLY_DISPATCH_DISPATCHED && - core.pending_slot_state->active_mask.requires_sync_start()))); + core.pending_slot_state->task_attrs.requires_sync_start()))); SlotTransition t = decide_slot_transition( reg_task_id, reg_state, core.running_reg_task_id, core.pending_reg_task_id, pending_gated ); @@ -331,8 +331,8 @@ void SchedulerContext::check_running_cores_for_completion( // fanin / deferred-completion (which may also clear pending_slot_state), // matching L2's finish_time point. Independent of L2 swimlane level, so // it works in SIMPLER_DFX=0 builds; untagged tasks pay only the compare. - if (core.pending_slot_state->task->task_timing_slot != TASK_TIMING_SLOT_NONE) { - aicpu_task_timing_finish(core.pending_slot_state->task->task_timing_slot, thread_idx); + if (core.pending_slot_state->task_attrs.is_timed()) { + aicpu_task_timing_finish(core.pending_slot_state->task_attrs.timing_slot(), thread_idx); } complete_slot_task( *core.pending_slot_state, core.pending_reg_task_id, core.pending_subslot, thread_idx, core_id, hank, @@ -345,8 +345,8 @@ void SchedulerContext::check_running_cores_for_completion( cur_thread_completed++; } if (t.running_done) { - if (core.running_slot_state->task->task_timing_slot != TASK_TIMING_SLOT_NONE) { - aicpu_task_timing_finish(core.running_slot_state->task->task_timing_slot, thread_idx); + if (core.running_slot_state->task_attrs.is_timed()) { + aicpu_task_timing_finish(core.running_slot_state->task_attrs.timing_slot(), thread_idx); } complete_slot_task( *core.running_slot_state, core.running_reg_task_id, core.running_subslot, thread_idx, core_id, hank, @@ -363,7 +363,7 @@ void SchedulerContext::check_running_cores_for_completion( if (t.running_freed) { if (core.pending_slot_state != nullptr && !t.pending_done) { PTO2TaskSlotState *promoted = core.pending_slot_state; - bool sync_start_promote = pending_gated && promoted->active_mask.requires_sync_start(); + bool sync_start_promote = pending_gated && promoted->task_attrs.requires_sync_start(); promote_pending_to_running(core); if (sync_start_promote) { promoted->payload->running_slot_count.fetch_add(1, std::memory_order_seq_cst); diff --git a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp index 864f8581b..3d3e08273 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp +++ b/src/a5/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_dispatch.cpp @@ -212,7 +212,7 @@ SchedulerContext::PublishHandle SchedulerContext::prepare_subtask_to_core( #endif return PublishHandle{ - core_exec_state.reg_addr, reg_task_id, core_offset, dispatch_timestamp_slot, slot_state.task->task_timing_slot + core_exec_state.reg_addr, reg_task_id, core_offset, dispatch_timestamp_slot, slot_state.task_attrs.timing_slot() }; } @@ -329,7 +329,7 @@ void SchedulerContext::dispatch_shape( // one register write. bool any_sync_start = false; for (int bi = 0; bi < got; bi++) { - if (batch[bi]->active_mask.requires_sync_start()) { + if (batch[bi]->task_attrs.requires_sync_start()) { any_sync_start = true; break; } @@ -387,7 +387,7 @@ void SchedulerContext::dispatch_shape( } } - if (slot_state->active_mask.requires_sync_start()) { + if (slot_state->task_attrs.requires_sync_start()) { if (is_pending) { sched_->ready_queues[static_cast(shape)].push(slot_state); continue; @@ -598,8 +598,8 @@ int32_t SchedulerContext::try_early_dispatch( // Tier 0: sync_start early cohorts (shape-agnostic, all-or-nothing drain). uint64_t sync_task_id_snapshot = 0; if (PTO2TaskSlotState *c = sched_->early_sync_start_queue.pop_tagged(&sync_task_id_snapshot)) { - bool current_sync_task = static_cast(c->task->task_id.raw) == sync_task_id_snapshot && - c->active_mask.requires_sync_start(); + bool current_sync_task = + static_cast(c->task->task_id.raw) == sync_task_id_snapshot && c->task_attrs.requires_sync_start(); if (current_sync_task && PTO2SchedulerState::try_claim_early_sync_drain(*c->payload)) { if (c->payload->early_dispatch_state.load(std::memory_order_seq_cst) != PTO2_EARLY_DISPATCH_STAGING) { sched_->cancel_early_sync_drain(*c);