Fix: bind AICore run streams to code images

python/simpler/task_interface.py

@@ -1264,7 +1264,7 @@ def host_dlopen_count(self):
 
     @property
     def run_stream_set_create_count(self):
-        """Number of run stream sets the bound runner has created."""
+        """Number of run stream generations the bound runner has created."""
         return self._impl.run_stream_set_create_count
 
     def malloc(self, size):

python/simpler/worker.py

@@ -6155,12 +6155,12 @@ def host_dlopen_count(self) -> int:
 
     @property
     def run_stream_set_create_count(self) -> int:
-        """L2 only: number of run stream sets the bound runner has created.
+        """L2 only: number of run stream generations the runner has created.
 
-        A set belongs to a pipeline slot and is reused for every run on that
-        slot, so a worker that has served any number of runs reports 1.
-        Returns 0 on non-L2 workers and on platforms whose runs use the
-        persistent bootstrap stream pair (simulation, a5).
+        AICPU streams belong to pipeline slots. AICore streams are reused only
+        while the loaded AICore image is unchanged, so each code transition
+        advances this count. Returns 0 on non-L2 workers and on platforms whose
+        runs use the persistent bootstrap stream pair (simulation, a5).
         """
         if self.level != 2 or self._chip_worker is None:
             return 0

src/a2a3/platform/onboard/host/device_runner.cpp

@@ -248,9 +248,16 @@ int DeviceRunner::run(Runtime &runtime, const CallConfig &config) {
         return rc;
     }
 
-    rc = ensure_run_stream_set(kPipelineSlot);
+    const int32_t callable_id = runtime.get_active_callable_id();
+    auto callable_it = callables_.find(callable_id);
+    if (callable_it == callables_.end()) {
+        LOG_ERROR("run() has no registered state for callable_id=%d", callable_id);
+        return -1;
+    }
+    const uint64_t aicore_image_hash = callable_it->second.aicore_image_hash;
+    rc = ensure_run_stream_set(kPipelineSlot, aicore_image_hash);
     if (rc != 0) {
-        LOG_ERROR("ensure_run_stream_set(%u) failed: %d", kPipelineSlot, rc);
+        LOG_ERROR("ensure_run_stream_set(%u, image=0x%lx) failed: %d", kPipelineSlot, aicore_image_hash, rc);
         return rc;
     }
 
@@ -480,19 +487,12 @@ int DeviceRunner::run(Runtime &runtime, const CallConfig &config) {
     return 0;
 }
 
-int DeviceRunner::ensure_run_stream_set(unsigned slot) {
+int DeviceRunner::ensure_run_stream_set(unsigned slot, uint64_t aicore_image_hash) {
     if (slot >= kRunStreamSetCount) {
         LOG_ERROR("ensure_run_stream_set: invalid slot %u", slot);
         return -1;
     }
     RunStreamSet &streams = run_stream_sets_[slot];
-    if (streams.aicpu != nullptr && streams.aicore != nullptr) {
-        return 0;
-    }
-
-    // Reached only on the first run for this slot, or after a partial create
-    // rolled one stream back; rtStreamCreate costs ~300 us and the set carries
-    // no per-run content, so it must not be rebuilt per run.
     if (streams.aicpu == nullptr) {
         int rc = rtStreamCreate(&streams.aicpu, 0);
         if (rc != 0) {
@@ -501,16 +501,31 @@ int DeviceRunner::ensure_run_stream_set(unsigned slot) {
             return rc;
         }
     }
-    if (streams.aicore == nullptr) {
-        int rc = rtStreamCreate(&streams.aicore, 0);
+    if (streams.aicore != nullptr && streams.has_aicore_image && streams.aicore_image_hash == aicore_image_hash) {
+        return 0;
+    }
+
+    if (streams.aicore != nullptr) {
+        int rc = rtStreamDestroy(streams.aicore);
         if (rc != 0) {
-            LOG_ERROR("rtStreamCreate (run AICore slot %u) failed: %d", slot, rc);
-            ACL_LOG_ERROR_DETAIL(rc);
+            LOG_ERROR("rtStreamDestroy (retired AICore slot %u) failed: %d", slot, rc);
             return rc;
         }
+        streams.aicore = nullptr;
+        streams.has_aicore_image = false;
+    }
+
+    int rc = rtStreamCreate(&streams.aicore, 0);
+    if (rc != 0) {
+        LOG_ERROR("rtStreamCreate (run AICore slot %u) failed: %d", slot, rc);
+        ACL_LOG_ERROR_DETAIL(rc);
+        streams.aicore = nullptr;
+        return rc;
     }
+    streams.aicore_image_hash = aicore_image_hash;
+    streams.has_aicore_image = true;
     ++run_stream_sets_created_;
-    LOG_INFO_V0("DeviceRunner: run stream set %u created", slot);
+    LOG_INFO_V0("DeviceRunner: run stream generation %zu created for slot %u", run_stream_sets_created_, slot);
     return 0;
 }
 
@@ -539,6 +554,7 @@ int DeviceRunner::destroy_run_stream_sets() {
             }
             capture(destroy_rc);
             streams.aicore = nullptr;
+            streams.has_aicore_image = false;
         }
     }
     return rc;

src/a2a3/platform/onboard/host/device_runner.h

@@ -228,17 +228,18 @@ class DeviceRunner : public DeviceRunnerBase {
     struct RunStreamSet {
         rtStream_t aicpu{nullptr};
         rtStream_t aicore{nullptr};
+        uint64_t aicore_image_hash{0};
+        bool has_aicore_image{false};
     };
     // One slot per in-flight run the pipeline contract can declare.
     static constexpr unsigned kRunStreamSetCount = PTO_PIPELINE_MAX_DEPTH;
 
-    // A stream set carries no per-run content, so it is created on first use
-    // and reused for every later run on this slot; `destroy_run_stream_sets()`
-    // in finalize() is the sole release point, as for the persistent bootstrap
-    // pair. Only slot 0 is selected while the contract is K=1.
+    // AICPU streams belong to slots. AICore streams additionally belong to the
+    // loaded code image because the platform has no explicit instruction-cache
+    // invalidation operation for code replaced in GM.
     RunStreamSet run_stream_sets_[kRunStreamSetCount]{};
     size_t run_stream_sets_created_{0};
-    int ensure_run_stream_set(unsigned slot);
+    int ensure_run_stream_set(unsigned slot, uint64_t aicore_image_hash);
     int destroy_run_stream_sets();
 
     // The kernel submission boundary is separate from the stream wait and the

src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp

@@ -558,7 +558,8 @@ register_callable_impl(const ChipCallable *callable, uint64_t (*upload_fn)(const
 
     LOG_INFO_V0("Registering %d kernel(s) in register_callable_impl", callable->child_count());
     if (upload_and_collect_child_addrs(
-            callable, upload_fn, &out->kernel_addrs, &out->chip_buffer_dev, &out->chip_buffer_hash
+            callable, upload_fn, &out->kernel_addrs, &out->chip_buffer_dev, &out->chip_buffer_hash,
+            &out->aicore_image_hash
         ) != 0) {
         LOG_ERROR("Failed to upload ChipCallable buffer");
         return -1;

src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp

@@ -416,7 +416,8 @@ register_callable_impl(const ChipCallable *callable, uint64_t (*upload_fn)(const
 
     LOG_INFO_V0("Registering %d kernel(s) in register_callable_impl", callable->child_count());
     if (upload_and_collect_child_addrs(
-            callable, upload_fn, &out->kernel_addrs, &out->chip_buffer_dev, &out->chip_buffer_hash
+            callable, upload_fn, &out->kernel_addrs, &out->chip_buffer_dev, &out->chip_buffer_hash,
+            &out->aicore_image_hash
         ) != 0) {
         LOG_ERROR("Failed to upload ChipCallable buffer");
         return -1;

src/a5/runtime/host_build_graph/host/runtime_maker.cpp

@@ -322,7 +322,8 @@ int register_callable_impl(const ChipCallable *callable, uint64_t (*upload_fn)(c
 
     LOG_INFO_V0("Registering %d kernel(s) in register_callable_impl", callable->child_count());
     if (upload_and_collect_child_addrs(
-            callable, upload_fn, &out->kernel_addrs, &out->chip_buffer_dev, &out->chip_buffer_hash
+            callable, upload_fn, &out->kernel_addrs, &out->chip_buffer_dev, &out->chip_buffer_hash,
+            &out->aicore_image_hash
         ) != 0) {
         LOG_ERROR("Failed to upload ChipCallable buffer");
         return -1;

src/a5/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp

@@ -393,7 +393,8 @@ register_callable_impl(const ChipCallable *callable, uint64_t (*upload_fn)(const
 
     LOG_INFO_V0("Registering %d kernel(s) in register_callable_impl", callable->child_count());
     if (upload_and_collect_child_addrs(
-            callable, upload_fn, &out->kernel_addrs, &out->chip_buffer_dev, &out->chip_buffer_hash
+            callable, upload_fn, &out->kernel_addrs, &out->chip_buffer_dev, &out->chip_buffer_hash,
+            &out->aicore_image_hash
         ) != 0) {
         LOG_ERROR("Failed to upload ChipCallable buffer");
         return -1;

src/common/platform/onboard/host/c_api_shared.cpp

@@ -434,17 +434,17 @@ int simpler_register_callable(DeviceContextHandle ctx, int32_t callable_id, cons
         bool needs_aicpu_register = false;
         if (artifacts.host_dlopen_handle != nullptr) {
             rc = runner->record_host_orch_callable(
-                callable_id, artifacts.chip_buffer_hash, artifacts.host_dlopen_handle, artifacts.host_orch_func_ptr,
-                std::move(kernel_addrs), std::move(artifacts.signature)
+                callable_id, artifacts.chip_buffer_hash, artifacts.aicore_image_hash, artifacts.host_dlopen_handle,
+                artifacts.host_orch_func_ptr, std::move(kernel_addrs), std::move(artifacts.signature)
             );
             if (rc != 0) return rc;
             host_dlopen_guard.dismiss();
             chip_buffer_guard.dismiss();
         } else {
             rc = runner->record_device_orch_callable(
-                callable_id, artifacts.chip_buffer_hash, artifacts.chip_buffer_dev, artifacts.orch_so_data,
-                artifacts.orch_so_size, artifacts.func_name.c_str(), artifacts.config_name.c_str(),
-                std::move(kernel_addrs), std::move(artifacts.signature)
+                callable_id, artifacts.chip_buffer_hash, artifacts.aicore_image_hash, artifacts.chip_buffer_dev,
+                artifacts.orch_so_data, artifacts.orch_so_size, artifacts.func_name.c_str(),
+                artifacts.config_name.c_str(), std::move(kernel_addrs), std::move(artifacts.signature)
             );
             if (rc != 0) return rc;
             chip_buffer_guard.dismiss();

src/common/platform/onboard/host/device_runner_base.cpp

@@ -735,9 +735,9 @@ int DeviceRunnerBase::launch_device_register(int32_t callable_id) {
 }
 
 int DeviceRunnerBase::record_device_orch_callable(
-    int32_t callable_id, uint64_t chip_buffer_hash, uint64_t chip_dev, const void *orch_so_data, size_t orch_so_size,
-    const char *func_name, const char *config_name, std::vector<std::pair<int, uint64_t>> kernel_addrs,
-    std::vector<ArgDirection> signature
+    int32_t callable_id, uint64_t chip_buffer_hash, uint64_t aicore_image_hash, uint64_t chip_dev,
+    const void *orch_so_data, size_t orch_so_size, const char *func_name, const char *config_name,
+    std::vector<std::pair<int, uint64_t>> kernel_addrs, std::vector<ArgDirection> signature
 ) {
     // The AICPU executor reserves `orch_so_table_[MAX_REGISTERED_CALLABLE_IDS]`
     // (declared in src/common/task_interface/callable_protocol.h) and indexes
@@ -767,6 +767,7 @@ int DeviceRunnerBase::record_device_orch_callable(
     CallableState state;
     state.hash = hash;
     state.chip_buffer_hash = chip_buffer_hash;
+    state.aicore_image_hash = aicore_image_hash;
     state.dev_orch_so_addr = chip_dev + offsetof(ChipCallable, storage_);
     state.dev_orch_so_size = orch_so_size;
     state.func_name = (func_name != nullptr) ? func_name : "";
@@ -782,8 +783,8 @@ int DeviceRunnerBase::record_device_orch_callable(
 }
 
 int DeviceRunnerBase::record_host_orch_callable(
-    int32_t callable_id, uint64_t chip_buffer_hash, void *host_dlopen_handle, void *host_orch_func_ptr,
-    std::vector<std::pair<int, uint64_t>> kernel_addrs, std::vector<ArgDirection> signature
+    int32_t callable_id, uint64_t chip_buffer_hash, uint64_t aicore_image_hash, void *host_dlopen_handle,
+    void *host_orch_func_ptr, std::vector<std::pair<int, uint64_t>> kernel_addrs, std::vector<ArgDirection> signature
 ) {
     if (callable_id < 0 || callable_id >= MAX_REGISTERED_CALLABLE_IDS) {
         LOG_ERROR(
@@ -806,6 +807,7 @@ int DeviceRunnerBase::record_host_orch_callable(
 
     CallableState state;
     state.chip_buffer_hash = chip_buffer_hash;
+    state.aicore_image_hash = aicore_image_hash;
     state.host_dlopen_handle = host_dlopen_handle;
     state.host_orch_func_ptr = host_orch_func_ptr;
     state.kernel_addrs = std::move(kernel_addrs);

src/common/platform/onboard/host/device_runner_base.h

@@ -303,8 +303,8 @@ class DeviceRunnerBase {
      * @return 0 on success, negative on failure.
      */
     int record_device_orch_callable(
-        int32_t callable_id, uint64_t chip_buffer_hash, uint64_t chip_dev, const void *orch_so_data,
-        size_t orch_so_size, const char *func_name, const char *config_name,
+        int32_t callable_id, uint64_t chip_buffer_hash, uint64_t aicore_image_hash, uint64_t chip_dev,
+        const void *orch_so_data, size_t orch_so_size, const char *func_name, const char *config_name,
         std::vector<std::pair<int, uint64_t>> kernel_addrs, std::vector<ArgDirection> signature
     );
 
@@ -319,8 +319,9 @@ class DeviceRunnerBase {
      * dlclose'd by `unregister_callable`. Increments `host_dlopen_total_`.
      */
     int record_host_orch_callable(
-        int32_t callable_id, uint64_t chip_buffer_hash, void *host_dlopen_handle, void *host_orch_func_ptr,
-        std::vector<std::pair<int, uint64_t>> kernel_addrs, std::vector<ArgDirection> signature
+        int32_t callable_id, uint64_t chip_buffer_hash, uint64_t aicore_image_hash, void *host_dlopen_handle,
+        void *host_orch_func_ptr, std::vector<std::pair<int, uint64_t>> kernel_addrs,
+        std::vector<ArgDirection> signature
     );
 
     /**
@@ -420,10 +421,9 @@ class DeviceRunnerBase {
     size_t host_dlopen_count() const { return host_dlopen_total_; }
 
     /**
-     * Number of run stream sets this runner has created. A set belongs to a
-     * pipeline slot and is reused for every run on that slot, so a runner that
-     * has served any number of runs on one slot reports 1. Arches whose runs
-     * use the persistent pair report 0.
+     * Number of run stream generations this runner has created. AICPU streams
+     * belong to pipeline slots, while an AICore stream is reused only for the
+     * same AICore image. Arches whose runs use the persistent pair report 0.
      */
     virtual size_t run_stream_set_create_count() const { return 0; }
 
@@ -806,6 +806,7 @@ class DeviceRunnerBase {
         // chip_buffer_hash, which keys the retained buffer.
         uint64_t hash{0};
         uint64_t chip_buffer_hash{0};
+        uint64_t aicore_image_hash{0};
         uint64_t dev_orch_so_addr{0};
         size_t dev_orch_so_size{0};
         std::string func_name;

src/common/task_interface/chip_callable_layout.h

@@ -30,9 +30,10 @@
 #include "utils/fnv1a_64.h"
 
 struct ChipCallableLayout {
-    size_t header_size;     // offsetof(ChipCallable, storage_)
-    size_t total_size;      // header_size + storage_used (matches make_callable())
-    uint64_t content_hash;  // FNV-1a 64 over [callable, total_size)
+    size_t header_size;          // offsetof(ChipCallable, storage_)
+    size_t total_size;           // header_size + storage_used (matches make_callable())
+    uint64_t content_hash;       // FNV-1a 64 over [callable, total_size)
+    uint64_t aicore_image_hash;  // FNV-1a 64 over func ids and child binaries
 };
 
 /**
@@ -45,15 +46,25 @@ struct ChipCallableLayout {
 inline ChipCallableLayout compute_chip_callable_layout(const ChipCallable *callable) {
     constexpr size_t kHeaderSize = offsetof(ChipCallable, storage_);
     size_t storage_used = static_cast<size_t>(callable->binary_size());
+    const int32_t child_count = callable->child_count();
+    uint64_t aicore_image_hash = simpler::common::utils::fnv1a_64(&child_count, sizeof(child_count));
     for (int32_t i = 0; i < callable->child_count(); ++i) {
         const CoreCallable &c = callable->child(i);
+        const int32_t func_id = callable->child_func_id(i);
+        const uint32_t binary_size = c.binary_size();
+        aicore_image_hash = simpler::common::utils::fnv1a_64_append(aicore_image_hash, &func_id, sizeof(func_id));
+        aicore_image_hash =
+            simpler::common::utils::fnv1a_64_append(aicore_image_hash, &binary_size, sizeof(binary_size));
+        aicore_image_hash = simpler::common::utils::fnv1a_64_append(
+            aicore_image_hash, c.binary_data(), static_cast<size_t>(binary_size)
+        );
         size_t child_total = CoreCallable::binary_data_offset() + static_cast<size_t>(c.binary_size());
         size_t end = static_cast<size_t>(callable->child_offset(i)) + child_total;
         if (end > storage_used) storage_used = end;
     }
     const size_t total_size = kHeaderSize + storage_used;
     const uint64_t hash = simpler::common::utils::fnv1a_64(reinterpret_cast<const uint8_t *>(callable), total_size);
-    return ChipCallableLayout{kHeaderSize, total_size, hash};
+    return ChipCallableLayout{kHeaderSize, total_size, hash, aicore_image_hash};
 }
 
 /**

src/common/task_interface/prepare_callable_common.h

@@ -65,6 +65,7 @@ struct CallableArtifacts {
     void *host_dlopen_handle{nullptr};  // hbg only
     void *host_orch_func_ptr{nullptr};  // hbg only
     uint64_t chip_buffer_hash{0};       // FNV-1a hash for the whole ChipCallable buffer
+    uint64_t aicore_image_hash{0};      // FNV-1a hash for func ids and AICore child binaries
     uint64_t chip_buffer_dev{0};        // device address of the ChipCallable header
     const void *orch_so_data{nullptr};  // trb only; host view used for validation/hash only
     size_t orch_so_size{0};             // trb only
@@ -90,7 +91,7 @@ struct CallableArtifacts {
  */
 inline int upload_and_collect_child_addrs(
     const ChipCallable *callable, uint64_t (*upload_fn)(const void *), std::vector<ChildKernelAddr> *out,
-    uint64_t *out_chip_dev = nullptr, uint64_t *out_chip_hash = nullptr
+    uint64_t *out_chip_dev = nullptr, uint64_t *out_chip_hash = nullptr, uint64_t *out_aicore_image_hash = nullptr
 ) {
     if (callable == nullptr || upload_fn == nullptr || out == nullptr) return -1;
     out->clear();
@@ -100,6 +101,7 @@ inline int upload_and_collect_child_addrs(
     if (chip_dev == 0) return -1;
     if (out_chip_dev != nullptr) *out_chip_dev = chip_dev;
     if (out_chip_hash != nullptr) *out_chip_hash = layout.content_hash;
+    if (out_aicore_image_hash != nullptr) *out_aicore_image_hash = layout.aicore_image_hash;
 
     out->reserve(static_cast<size_t>(callable->child_count()));
     for (int32_t i = 0; i < callable->child_count(); ++i) {

src/common/utils/fnv1a_64.h

@@ -20,15 +20,19 @@ namespace simpler::common::utils {
 // FNV-1a 64-bit content hash. Deterministic, allocation-free, ~µs / MB.
 // Used as a generic content-keyed dedup key (ChipCallable buffer hashing in
 // DeviceRunner, ELF Build-ID fallback in elf_build_id.h, etc.).
-inline uint64_t fnv1a_64(const void *data, std::size_t len) {
+inline uint64_t fnv1a_64_append(uint64_t hash, const void *data, std::size_t len) {
     constexpr uint64_t kPrime = 0x00000100000001b3ULL;
-    uint64_t h = 0xcbf29ce484222325ULL;
     const auto *p = static_cast<const uint8_t *>(data);
     for (std::size_t i = 0; i < len; ++i) {
-        h ^= p[i];
-        h *= kPrime;
+        hash ^= p[i];
+        hash *= kPrime;
     }
-    return h;
+    return hash;
+}
+
+inline uint64_t fnv1a_64(const void *data, std::size_t len) {
+    constexpr uint64_t kOffsetBasis = 0xcbf29ce484222325ULL;
+    return fnv1a_64_append(kOffsetBasis, data, len);
 }
 
 }  // namespace simpler::common::utils

src/common/worker/chip_worker.h

@@ -90,11 +90,9 @@ class ChipWorker {
     /// `aicpu_dlopen_count` for the trb path; returns 0 on device-orch variants.
     size_t host_dlopen_count() const;
 
-    /// Number of run stream sets the bound runner has created. A set belongs
-    /// to a pipeline slot and is reused for every run on that slot, so a
-    /// runner that has served any number of runs on one slot reports 1;
-    /// platforms whose runs use the persistent bootstrap pair report 0. Used
-    /// by tests to assert that repeated runs do not rebuild the set per run.
+    /// Number of run stream generations the bound runner has created. AICPU
+    /// streams belong to slots; AICore streams are reused only while the loaded
+    /// code image is unchanged. Platforms using the persistent pair report 0.
     size_t run_stream_set_create_count() const;
 
     uint64_t malloc(size_t size);